diff --git a/Cargo.toml b/Cargo.toml
index 086ef37e..430e7ef0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,7 +1,5 @@
 [workspace]
-
 resolver = "2"
-
 members = [
   "crates/*",
   "crates/optix/examples/ex*",
diff --git a/crates/cuda_builder/src/lib.rs b/crates/cuda_builder/src/lib.rs
index e5b1e60d..5317c005 100644
--- a/crates/cuda_builder/src/lib.rs
+++ b/crates/cuda_builder/src/lib.rs
@@ -376,10 +376,13 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result<PathBuf, CudaBuilderError> {
 
     let new_path = get_new_path_var();
 
-    let mut rustflags = vec![format!(
-        "-Zcodegen-backend={}",
-        rustc_codegen_nvvm.display(),
-    )];
+    let mut rustflags = vec![
+        format!("-Zcodegen-backend={}", rustc_codegen_nvvm.display()),
+        "-Zcrate-attr=feature(register_tool)".into(),
+        "-Zcrate-attr=register_tool(nvvm_internal)".into(),
+        "-Zcrate-attr=no_std".into(),
+        "-Zsaturating_float_casts=false".into(),
+    ];
 
     if let Some(emit) = &builder.emit {
         let string = match emit {
@@ -437,8 +440,7 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result<PathBuf, CudaBuilderError> {
         "--lib",
         "--message-format=json-render-diagnostics",
         "-Zbuild-std=core,alloc",
-        "--target",
-        "nvptx64-nvidia-cuda",
+        "--target=nvptx64-nvidia-cuda",
     ]);
 
     cargo.args(&builder.build_args);
diff --git a/crates/cuda_std/src/atomic/intrinsics.rs b/crates/cuda_std/src/atomic/intrinsics.rs
index ba8565fe..7a4fc2a8 100644
--- a/crates/cuda_std/src/atomic/intrinsics.rs
+++ b/crates/cuda_std/src/atomic/intrinsics.rs
@@ -1,6 +1,7 @@
 //! Raw CUDA-specific atomic functions that map to PTX instructions.
 
 use crate::gpu_only;
+use core::arch::asm;
 use core::concat;
 use paste::paste;
 
diff --git a/crates/cuda_std/src/float.rs b/crates/cuda_std/src/float.rs
index 8b657227..c4a4505c 100644
--- a/crates/cuda_std/src/float.rs
+++ b/crates/cuda_std/src/float.rs
@@ -6,43 +6,81 @@
 /// std float intrinsics implemented using libdevice intrinsics so they can be used
 /// from GPU no_std crates. Falls back to stdlib implementation on non-nvptx.
 pub trait GpuFloat: Copy + PartialOrd + private::Sealed {
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn floor(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn ceil(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn round(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn trunc(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn fract(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn abs(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn signum(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn copysign(self, sign: Self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn mul_add(self, a: Self, b: Self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn div_euclid(self, rhs: Self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn rem_euclid(self, rhs: Self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn powi(self, n: i32) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn powf(self, n: Self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn sqrt(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn exp(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn exp2(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn ln(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn log(self, base: Self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn log2(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn log10(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn cbrt(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn hypot(self, other: Self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn sin(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn cos(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn tan(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn asin(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn acos(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn atan(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn atan2(self, other: Self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn sin_cos(self) -> (Self, Self);
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn exp_m1(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn ln_1p(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn sinh(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn cosh(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn tanh(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn asinh(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn acosh(self) -> Self;
+    #[must_use = "method returns a new number and does not mutate the original value"]
     fn atanh(self) -> Self;
 }
 
@@ -77,14 +115,12 @@ use crate::intrinsics;
 
 impl GpuFloat for f32 {
     /// Returns the largest integer less than or equal to a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn floor(self) -> f32 {
         f32_intrinsic!(self, floor())
     }
 
     /// Returns the smallest integer greater than or equal to a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn ceil(self) -> f32 {
         f32_intrinsic!(self, ceil())
@@ -92,21 +128,18 @@ impl GpuFloat for f32 {
 
     /// Returns the nearest integer to a number. Round half-way cases away from
     /// `0.0`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn round(self) -> f32 {
         f32_intrinsic!(self, round())
     }
 
     /// Returns the integer part of a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn trunc(self) -> f32 {
         f32_intrinsic!(self, trunc())
     }
 
     /// Returns the fractional part of a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn fract(self) -> f32 {
         self - self.trunc()
@@ -114,7 +147,6 @@ impl GpuFloat for f32 {
 
     /// Computes the absolute value of `self`. Returns `NAN` if the
     /// number is `NAN`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn abs(self) -> f32 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -129,7 +161,6 @@ impl GpuFloat for f32 {
     /// - `1.0` if the number is positive, `+0.0` or `INFINITY`
     /// - `-1.0` if the number is negative, `-0.0` or `NEG_INFINITY`
     /// - `NAN` if the number is `NAN`intrinsics
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn signum(self) -> f32 {
         if self.is_nan() {
@@ -145,7 +176,6 @@ impl GpuFloat for f32 {
     /// Equal to `self` if the sign of `self` and `sign` are the same, otherwise
     /// equal to `-self`. If `self` is a `NAN`, then a `NAN` with the sign of
     /// `sign` is returned.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn copysign(self, sign: f32) -> f32 {
         f32_intrinsic!(self, copysign(sign))
@@ -158,7 +188,6 @@ impl GpuFloat for f32 {
     /// the target architecture has a dedicated `fma` CPU instruction. However,
     /// this is not always true, and will be heavily dependant on designing
     /// algorithms with specific target hardware in mind.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn mul_add(self, a: f32, b: f32) -> f32 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -174,7 +203,6 @@ impl GpuFloat for f32 {
     /// `self = n * rhs + self.rem_euclid(rhs)`.
     /// In other words, the result is `self / rhs` rounded to the integer `n`
     /// such that `self >= n * rhs`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn div_euclid(self, rhs: f32) -> f32 {
         let q = (self / rhs).trunc();
@@ -194,7 +222,6 @@ impl GpuFloat for f32 {
     /// closest floating point number in the real numbers and thus fulfills the
     /// property `self == self.div_euclid(rhs) * rhs + self.rem_euclid(rhs)`
     /// approximatively.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn rem_euclid(self, rhs: f32) -> f32 {
         let r = self % rhs;
@@ -208,14 +235,12 @@ impl GpuFloat for f32 {
     /// Raises a number to an integer power.
     ///
     /// Using this function is generally faster than using `powf`intrinsics
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn powi(self, n: i32) -> f32 {
         f32_intrinsic!(self, powi(n))
     }
 
     /// Raises a number to a floating point power.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn powf(self, n: f32) -> f32 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -228,28 +253,24 @@ impl GpuFloat for f32 {
     /// Returns the square root of a number.
     ///
     /// Returns NaN if `self` is a negative number other than `-0.0`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn sqrt(self) -> f32 {
         f32_intrinsic!(self, sqrt())
     }
 
     /// Returns `e^(self)`, (the exponential function).
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn exp(self) -> f32 {
         f32_intrinsic!(self, exp())
     }
 
     /// Returns `2^(self)`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn exp2(self) -> f32 {
         f32_intrinsic!(self, exp2())
     }
 
     /// Returns the natural logarithm of the number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn ln(self) -> f32 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -264,28 +285,24 @@ impl GpuFloat for f32 {
     /// The result might not be correctly rounded owing to implementation details;
     /// `self.log2()` can produce more accurate results for base 2, and
     /// `self.log10()` can produce more accurate results for base 10.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn log(self, base: f32) -> f32 {
         self.ln() / base.ln()
     }
 
     /// Returns the base 2 logarithm of the number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn log2(self) -> f32 {
         f32_intrinsic!(self, log10())
     }
 
     /// Returns the base 10 logarithm of the number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn log10(self) -> f32 {
         f32_intrinsic!(self, log10())
     }
 
     /// Returns the cube root of a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn cbrt(self) -> f32 {
         f32_intrinsic!(self, cbrt())
@@ -293,28 +310,24 @@ impl GpuFloat for f32 {
 
     /// Calculates the length of the hypotenuse of a right-angle triangle given
     /// legs of length `x` and `y`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn hypot(self, other: f32) -> f32 {
         f32_intrinsic!(self, hypot(other))
     }
 
     /// Computes the sine of a number (in radians).
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn sin(self) -> f32 {
         f32_intrinsic!(self, sin())
     }
 
     /// Computes the cosine of a number (in radians).
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn cos(self) -> f32 {
         f32_intrinsic!(self, cos())
     }
 
     /// Computes the tangent of a number (in radians).
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn tan(self) -> f32 {
         f32_intrinsic!(self, tan())
@@ -323,7 +336,6 @@ impl GpuFloat for f32 {
     /// Computes the arcsine of a number. Return value is in radians in
     /// the range [-pi/2, pi/2] or NaN if the number is outside the range
     /// [-1, 1].
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn asin(self) -> f32 {
         f32_intrinsic!(self, asin())
@@ -332,7 +344,6 @@ impl GpuFloat for f32 {
     /// Computes the arccosine of a number. Return value is in radians in
     /// the range [0, pi] or NaN if the number is outside the range
     /// [-1, 1].
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn acos(self) -> f32 {
         f32_intrinsic!(self, acos())
@@ -340,7 +351,6 @@ impl GpuFloat for f32 {
 
     /// Computes the arctangent of a number. Return value is in radians in the
     /// range [-pi/2, pi/2];intrinsics
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn atan(self) -> f32 {
         f32_intrinsic!(self, atan())
@@ -352,7 +362,6 @@ impl GpuFloat for f32 {
     /// * `x >= 0`: `arctan(y/x)` -> `[-pi/2, pi/2]`
     /// * `y >= 0`: `arctan(y/x) + pi` -> `(pi/2, pi]`
     /// * `y < 0`: `arctan(y/x) - pi` -> `(-pi, -pi/2)`intrinsics
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn atan2(self, other: f32) -> f32 {
         f32_intrinsic!(self, atan2(other))
@@ -378,7 +387,6 @@ impl GpuFloat for f32 {
 
     /// Returns `e^(self) - 1` in a way that is accurate even if the
     /// number is close to zero.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn exp_m1(self) -> f32 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -390,7 +398,6 @@ impl GpuFloat for f32 {
 
     /// Returns `ln(1+n)` (natural logarithm) more accurately than if
     /// the operations were performed separately.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn ln_1p(self) -> f32 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -401,42 +408,36 @@ impl GpuFloat for f32 {
     }
 
     /// Hyperbolic sine function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn sinh(self) -> f32 {
         f32_intrinsic!(self, sinh())
     }
 
     /// Hyperbolic cosine function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn cosh(self) -> f32 {
         f32_intrinsic!(self, cosh())
     }
 
     /// Hyperbolic tangent function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn tanh(self) -> f32 {
         f32_intrinsic!(self, tanh())
     }
 
     /// Inverse hyperbolic sine function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn asinh(self) -> f32 {
         f32_intrinsic!(self, asinh())
     }
 
     /// Inverse hyperbolic cosine function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn acosh(self) -> f32 {
         f32_intrinsic!(self, acosh())
     }
 
     /// Inverse hyperbolic tangent function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn atanh(self) -> f32 {
         f32_intrinsic!(self, atanh())
@@ -445,14 +446,12 @@ impl GpuFloat for f32 {
 
 impl GpuFloat for f64 {
     /// Returns the largest integer less than or equal to a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn floor(self) -> f64 {
         f64_intrinsic!(self, floor())
     }
 
     /// Returns the smallest integer greater than or equal to a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn ceil(self) -> f64 {
         f64_intrinsic!(self, ceil())
@@ -460,21 +459,18 @@ impl GpuFloat for f64 {
 
     /// Returns the nearest integer to a number. Round half-way cases away from
     /// `0.0`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn round(self) -> f64 {
         f64_intrinsic!(self, round())
     }
 
     /// Returns the integer part of a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn trunc(self) -> f64 {
         f64_intrinsic!(self, trunc())
     }
 
     /// Returns the fractional part of a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn fract(self) -> f64 {
         self - self.trunc()
@@ -482,7 +478,6 @@ impl GpuFloat for f64 {
 
     /// Computes the absolute value of `self`. Returns `NAN` if the
     /// number is `NAN`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn abs(self) -> f64 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -497,7 +492,6 @@ impl GpuFloat for f64 {
     /// - `1.0` if the number is positive, `+0.0` or `INFINITY`
     /// - `-1.0` if the number is negative, `-0.0` or `NEG_INFINITY`
     /// - `NAN` if the number is `NAN`intrinsics
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn signum(self) -> f64 {
         if self.is_nan() {
@@ -513,7 +507,6 @@ impl GpuFloat for f64 {
     /// Equal to `self` if the sign of `self` and `sign` are the same, otherwise
     /// equal to `-self`. If `self` is a `NAN`, then a `NAN` with the sign of
     /// `sign` is returned.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn copysign(self, sign: f64) -> f64 {
         f64_intrinsic!(self, copysign(sign))
@@ -526,7 +519,6 @@ impl GpuFloat for f64 {
     /// the target architecture has a dedicated `fma` CPU instruction. However,
     /// this is not always true, and will be heavily dependant on designing
     /// algorithms with specific target hardware in mind.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn mul_add(self, a: f64, b: f64) -> f64 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -542,7 +534,6 @@ impl GpuFloat for f64 {
     /// `self = n * rhs + self.rem_euclid(rhs)`.
     /// In other words, the result is `self / rhs` rounded to the integer `n`
     /// such that `self >= n * rhs`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn div_euclid(self, rhs: f64) -> f64 {
         let q = (self / rhs).trunc();
@@ -562,7 +553,6 @@ impl GpuFloat for f64 {
     /// closest floating point number in the real numbers and thus fulfills the
     /// property `self == self.div_euclid(rhs) * rhs + self.rem_euclid(rhs)`
     /// approximatively.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn rem_euclid(self, rhs: f64) -> f64 {
         let r = self % rhs;
@@ -576,14 +566,12 @@ impl GpuFloat for f64 {
     /// Raises a number to an integer power.
     ///
     /// Using this function is generally faster than using `powf`intrinsics
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn powi(self, n: i32) -> f64 {
         f64_intrinsic!(self, powi(n))
     }
 
     /// Raises a number to a floating point power.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn powf(self, n: f64) -> f64 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -596,28 +584,24 @@ impl GpuFloat for f64 {
     /// Returns the square root of a number.
     ///
     /// Returns NaN if `self` is a negative number other than `-0.0`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn sqrt(self) -> f64 {
         f64_intrinsic!(self, sqrt())
     }
 
     /// Returns `e^(self)`, (the exponential function).
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn exp(self) -> f64 {
         f64_intrinsic!(self, exp())
     }
 
     /// Returns `2^(self)`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn exp2(self) -> f64 {
         f64_intrinsic!(self, exp2())
     }
 
     /// Returns the natural logarithm of the number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn ln(self) -> f64 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -632,28 +616,24 @@ impl GpuFloat for f64 {
     /// The result might not be correctly rounded owing to implementation details;
     /// `self.log2()` can produce more accurate results for base 2, and
     /// `self.log10()` can produce more accurate results for base 10.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn log(self, base: f64) -> f64 {
         self.ln() / base.ln()
     }
 
     /// Returns the base 2 logarithm of the number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn log2(self) -> f64 {
         f64_intrinsic!(self, log10())
     }
 
     /// Returns the base 10 logarithm of the number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn log10(self) -> f64 {
         f64_intrinsic!(self, log10())
     }
 
     /// Returns the cube root of a number.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn cbrt(self) -> f64 {
         f64_intrinsic!(self, cbrt())
@@ -661,28 +641,24 @@ impl GpuFloat for f64 {
 
     /// Calculates the length of the hypotenuse of a right-angle triangle given
     /// legs of length `x` and `y`.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn hypot(self, other: f64) -> f64 {
         f64_intrinsic!(self, hypot(other))
     }
 
     /// Computes the sine of a number (in radians).
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn sin(self) -> f64 {
         f64_intrinsic!(self, sin())
     }
 
     /// Computes the cosine of a number (in radians).
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn cos(self) -> f64 {
         f64_intrinsic!(self, cos())
     }
 
     /// Computes the tangent of a number (in radians).
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn tan(self) -> f64 {
         f64_intrinsic!(self, tan())
@@ -691,7 +667,6 @@ impl GpuFloat for f64 {
     /// Computes the arcsine of a number. Return value is in radians in
     /// the range [-pi/2, pi/2] or NaN if the number is outside the range
     /// [-1, 1].
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn asin(self) -> f64 {
         f64_intrinsic!(self, asin())
@@ -700,7 +675,6 @@ impl GpuFloat for f64 {
     /// Computes the arccosine of a number. Return value is in radians in
     /// the range [0, pi] or NaN if the number is outside the range
     /// [-1, 1].
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn acos(self) -> f64 {
         f64_intrinsic!(self, acos())
@@ -708,7 +682,6 @@ impl GpuFloat for f64 {
 
     /// Computes the arctangent of a number. Return value is in radians in the
     /// range [-pi/2, pi/2];intrinsics
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn atan(self) -> f64 {
         f64_intrinsic!(self, atan())
@@ -720,7 +693,6 @@ impl GpuFloat for f64 {
     /// * `x >= 0`: `arctan(y/x)` -> `[-pi/2, pi/2]`
     /// * `y >= 0`: `arctan(y/x) + pi` -> `(pi/2, pi]`
     /// * `y < 0`: `arctan(y/x) - pi` -> `(-pi, -pi/2)`intrinsics
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn atan2(self, other: f64) -> f64 {
         f64_intrinsic!(self, atan2(other))
@@ -746,7 +718,6 @@ impl GpuFloat for f64 {
 
     /// Returns `e^(self) - 1` in a way that is accurate even if the
     /// number is close to zero.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn exp_m1(self) -> f64 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -758,7 +729,6 @@ impl GpuFloat for f64 {
 
     /// Returns `ln(1+n)` (natural logarithm) more accurately than if
     /// the operations were performed separately.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn ln_1p(self) -> f64 {
         #[cfg(not(target_arch = "nvptx64"))]
@@ -769,42 +739,36 @@ impl GpuFloat for f64 {
     }
 
     /// Hyperbolic sine function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn sinh(self) -> f64 {
         f64_intrinsic!(self, sinh())
     }
 
     /// Hyperbolic cosine function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn cosh(self) -> f64 {
         f64_intrinsic!(self, cosh())
     }
 
     /// Hyperbolic tangent function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn tanh(self) -> f64 {
         f64_intrinsic!(self, tanh())
     }
 
     /// Inverse hyperbolic sine function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn asinh(self) -> f64 {
         f64_intrinsic!(self, asinh())
     }
 
     /// Inverse hyperbolic cosine function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn acosh(self) -> f64 {
         f64_intrinsic!(self, acosh())
     }
 
     /// Inverse hyperbolic tangent function.
-    #[must_use = "method returns a new number and does not mutate the original value"]
     #[inline]
     fn atanh(self) -> f64 {
         f64_intrinsic!(self, atanh())
diff --git a/crates/cuda_std/src/lib.rs b/crates/cuda_std/src/lib.rs
index 5276bdec..357922b5 100644
--- a/crates/cuda_std/src/lib.rs
+++ b/crates/cuda_std/src/lib.rs
@@ -5,7 +5,7 @@
 //!
 //! This library will build on non-nvptx targets or targets not using the nvvm backend. However, it will not
 //! be usable, and it will throw linker errors if you attempt to use most of the functions in the library.
-//! However, [`kernel`] automatically cfg-gates the function annotated for `nvptx64` or `nvptx`, therefore,
+//! However, [`kernel`] automatically cfg-gates the function annotated for `nvptx64`, therefore,
 //! no "actual" functions from this crate should be used when compiling for a non-nvptx target.
 //!
 //! This crate cannot be used with the llvm ptx backend either, it heavily relies on external functions implicitly
@@ -21,17 +21,10 @@
 //! In order to simplify imports, we provide a prelude module which contains GPU analogues to standard library
 //! structures as well as common imports such as [`thread`].
 
+#![allow(internal_features)]
 #![cfg_attr(
     target_os = "cuda",
-    no_std,
-    feature(
-        register_attr,
-        alloc_error_handler,
-        asm,
-        asm_experimental_arch,
-        link_llvm_intrinsics
-    ),
-    register_attr(nvvm_internal)
+    feature(alloc_error_handler, asm_experimental_arch, link_llvm_intrinsics)
 )]
 
 extern crate alloc;
diff --git a/crates/cuda_std/src/mem.rs b/crates/cuda_std/src/mem.rs
index 3b345ce1..d1c9626b 100644
--- a/crates/cuda_std/src/mem.rs
+++ b/crates/cuda_std/src/mem.rs
@@ -37,7 +37,7 @@ pub static GLOBAL_ALLOCATOR: CUDAAllocator = CUDAAllocator;
 pub fn dynamic_smem_size() -> u32 {
     let mut out;
     unsafe {
-        asm!(
+        core::arch::asm!(
             "mov.u32 {}, %dynamic_smem_size",
             out(reg32) out
         )
@@ -55,7 +55,7 @@ pub fn dynamic_smem_size() -> u32 {
 pub fn total_smem_size() -> u32 {
     let mut out;
     unsafe {
-        asm!(
+        core::arch::asm!(
             "mov.u32 {}, %total_smem_size",
             out(reg32) out
         )
diff --git a/crates/cuda_std/src/misc.rs b/crates/cuda_std/src/misc.rs
index 9314a3ce..1d80b7be 100644
--- a/crates/cuda_std/src/misc.rs
+++ b/crates/cuda_std/src/misc.rs
@@ -1,6 +1,8 @@
 //! Misc functions that do not exactly fit into other categories.
 
 use crate::gpu_only;
+#[cfg(target_os = "cuda")]
+use core::arch::asm;
 
 /// Suspends execution of the kernel, usually to pause at a specific point when debugging in a debugger.
 #[gpu_only]
diff --git a/crates/cuda_std/src/ptr.rs b/crates/cuda_std/src/ptr.rs
index 02dd8767..3d1cac60 100644
--- a/crates/cuda_std/src/ptr.rs
+++ b/crates/cuda_std/src/ptr.rs
@@ -1,6 +1,8 @@
 //! CUDA-specific pointer handling logic.
 
 use crate::gpu_only;
+#[cfg(target_os = "cuda")]
+use core::arch::asm;
 
 /// Special areas of GPU memory where a pointer could reside.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
diff --git a/crates/cuda_std/src/shared.rs b/crates/cuda_std/src/shared.rs
index bbbefcfe..086746a9 100644
--- a/crates/cuda_std/src/shared.rs
+++ b/crates/cuda_std/src/shared.rs
@@ -19,11 +19,11 @@ use crate::gpu_only;
 /// the burden of correctness is on the user. Some of the things you must ensure in your usage of
 /// shared memory are:
 /// - Shared memory is only shared across __thread blocks__, not the entire device, therefore it is
-/// unsound to try and rely on sharing data across more than one block.
+///   unsound to try and rely on sharing data across more than one block.
 /// - You must write to the shared buffer before reading from it as the data is uninitialized by default.
 /// - [`thread::sync_threads`](crate::thread::sync_threads) must be called before relying on the results of other
-/// threads, this ensures every thread has reached that point before going on. For example, reading another thread's
-/// data after writing to the buffer.
+///   threads, this ensures every thread has reached that point before going on. For example, reading another thread's
+///   data after writing to the buffer.
 /// - No access may be out of bounds, this usually means making sure the amount of threads and their dimensions are correct.
 ///
 /// It is suggested to run your executable in `cuda-memcheck` to make sure usages of shared memory are right.
@@ -68,7 +68,7 @@ macro_rules! shared_array {
 /// Gets a pointer to the dynamic shared memory that was allocated by the caller of the kernel. The
 /// data is left uninitialized.
 ///
-/// **Calling this function multiple times will yield the same pointer**.  
+/// **Calling this function multiple times will yield the same pointer**.
 #[gpu_only]
 pub fn dynamic_shared_mem<T>() -> *mut T {
     // it is unclear whether an alignment of 16 is actually required for correctness, however,
@@ -77,7 +77,7 @@ pub fn dynamic_shared_mem<T>() -> *mut T {
     extern "C" {
         // need to use nvvm_internal and not address_space because address_space only parses
         // static definitions, not extern static definitions.
-        #[nvvm_internal(addrspace(3))]
+        #[nvvm_internal::addrspace(3)]
         #[allow(improper_ctypes)]
         // mangle it a bit to make sure nobody makes the same thing
         #[link_name = "_Zcuda_std_dyn_shared"]
diff --git a/crates/cuda_std/src/thread.rs b/crates/cuda_std/src/thread.rs
index e32d1bb6..1f70bbb5 100644
--- a/crates/cuda_std/src/thread.rs
+++ b/crates/cuda_std/src/thread.rs
@@ -344,7 +344,7 @@ pub fn system_fence() {
 #[inline(always)]
 pub fn nanosleep(nanos: u32) {
     unsafe {
-        asm!(
+        core::arch::asm!(
             "nanosleep {}",
             in(reg32) nanos
         )
diff --git a/crates/cuda_std/src/warp.rs b/crates/cuda_std/src/warp.rs
index dae087dd..46685e6a 100644
--- a/crates/cuda_std/src/warp.rs
+++ b/crates/cuda_std/src/warp.rs
@@ -4,6 +4,8 @@
 //! thread blocks and execute in SIMT fashion.
 
 use crate::gpu_only;
+#[cfg(target_os = "cuda")]
+use core::arch::asm;
 use half::{bf16, f16};
 
 /// Synchronizes all of the threads inside of this warp according to `mask`.
@@ -328,7 +330,7 @@ unsafe fn match_all_64(mask: u32, value: u64) -> (u32, bool) {
 /// Behavior is undefined if:
 /// - Any thread participating in the vote has exited or the executing thread is not in `mask`.
 /// - For `compute_62` and below, all threads in `mask` must call this function in convergence, and only threads belonging
-/// to the `mask` can be active when the intrinsic is called.
+///   to the `mask` can be active when the intrinsic is called.
 /// - A thread tries to execute this function while not being present in `mask`.
 #[gpu_only]
 pub unsafe fn warp_vote_all(mask: u32, predicate: bool) -> bool {
@@ -358,7 +360,7 @@ pub unsafe fn warp_vote_all(mask: u32, predicate: bool) -> bool {
 /// Behavior is undefined if:
 /// - Any thread participating in the vote has exited or the executing thread is not in `mask`.
 /// - For `compute_62` and below, all threads in `mask` must call this function in convergence, and only threads belonging
-/// to the `mask` can be active when the intrinsic is called.
+///   to the `mask` can be active when the intrinsic is called.
 /// - A thread tries to execute this function while not being present in `mask`.
 #[gpu_only]
 pub unsafe fn warp_vote_any(mask: u32, predicate: bool) -> bool {
@@ -388,7 +390,7 @@ pub unsafe fn warp_vote_any(mask: u32, predicate: bool) -> bool {
 /// Behavior is undefined if:
 /// - Any thread participating in the vote has exited or the executing thread is not in `mask`.
 /// - For `compute_62` and below, all threads in `mask` must call this function in convergence, and only threads belonging
-/// to the `mask` can be active when the intrinsic is called.
+///   to the `mask` can be active when the intrinsic is called.
 /// - A thread tries to execute this function while not being present in `mask`.
 #[gpu_only]
 pub unsafe fn warp_vote_ballot(mask: u32, predicate: bool) -> u32 {
@@ -414,10 +416,10 @@ pub unsafe fn warp_vote_ballot(mask: u32, predicate: bool) -> u32 {
 ///
 /// - `mask` dictates what threads will participate in the shuffle, usually [`u32::MAX`] to indicate all threads.
 /// - `value` is the value that will be shuffled across the threads. i.e. the value that will be given to the thread
-/// that calculates this thread as its target lane.
+///   that calculates this thread as its target lane.
 /// - `delta` is the value that will be subtracted from the current thread's lane to calculate the target lane.
 /// - `width` dictates how to optionally split the warp into subsections, it must be a power of two and lower than `32`.
-/// calculated source lane values will NOT wrap around the value of `width`. Usually just `32`.
+///   calculated source lane values will NOT wrap around the value of `width`. Usually just `32`.
 ///
 /// # Returns
 ///
@@ -438,7 +440,7 @@ pub unsafe fn warp_vote_ballot(mask: u32, predicate: bool) -> u32 {
 /// Behavior is undefined if:
 /// - Any thread participating in the shuffle has exited or the executing thread is not in `mask`.
 /// - For `compute_62` and below, all threads in `mask` must call the same function in convergence, and only the threads
-/// in `mask` can be active when the shuffle is called.
+///   in `mask` can be active when the shuffle is called.
 ///
 /// The returned value returned is unspecified if the calculated target lane is inactive.
 pub unsafe fn warp_shuffle_down<T: WarpShuffleValue>(
@@ -456,10 +458,10 @@ pub unsafe fn warp_shuffle_down<T: WarpShuffleValue>(
 ///
 /// - `mask` dictates what threads will participate in the shuffle, usually [`u32::MAX`] to indicate all threads.
 /// - `value` is the value that will be shuffled across the threads. i.e. the value that will be given to the thread
-/// that calculates this thread as its target lane.
+///   that calculates this thread as its target lane.
 /// - `delta` is the value that will be added to the current thread's lane to calculate the target lane.
 /// - `width` dictates how to optionally split the warp into subsections, it must be a power of two and lower than `32`.
-/// calculated source lane values will NOT wrap around the value of `width`. Usually just `32`.
+///   calculated source lane values will NOT wrap around the value of `width`. Usually just `32`.
 ///
 /// # Returns
 ///
@@ -480,7 +482,7 @@ pub unsafe fn warp_shuffle_down<T: WarpShuffleValue>(
 /// Behavior is undefined if:
 /// - Any thread participating in the shuffle has exited or the executing thread is not in `mask`.
 /// - For `compute_62` and below, all threads in `mask` must call the same function in convergence, and only the threads
-/// in `mask` can be active when the shuffle is called.
+///   in `mask` can be active when the shuffle is called.
 ///
 /// The returned value returned is unspecified if the calculated target lane is inactive.
 pub unsafe fn warp_shuffle_up<T: WarpShuffleValue>(
@@ -498,10 +500,10 @@ pub unsafe fn warp_shuffle_up<T: WarpShuffleValue>(
 ///
 /// - `mask` dictates what threads will participate in the shuffle, usually [`u32::MAX`] to indicate all threads.
 /// - `value` is the value that will be shuffled across the threads. i.e. the value that will be given to the thread
-/// that calculates this thread as its target lane.
+///   that calculates this thread as its target lane.
 /// - `idx` is the target lane that will be used as the source of this thread's returned value.
 /// - `width` dictates how to optionally split the warp into subsections, it must be a power of two and lower than `32`.
-/// calculated source lane values will NOT wrap around the value of `width`. Usually just `32`.
+///   calculated source lane values will NOT wrap around the value of `width`. Usually just `32`.
 ///
 /// # Returns
 ///
@@ -522,7 +524,7 @@ pub unsafe fn warp_shuffle_up<T: WarpShuffleValue>(
 /// Behavior is undefined if:
 /// - Any thread participating in the shuffle has exited or the executing thread is not in `mask`.
 /// - For `compute_62` and below, all threads in `mask` must call the same function in convergence, and only the threads
-/// in `mask` can be active when the shuffle is called.
+///   in `mask` can be active when the shuffle is called.
 ///
 /// The returned value returned is unspecified if the calculated target lane is inactive.
 pub unsafe fn warp_shuffle_idx<T: WarpShuffleValue>(
@@ -540,11 +542,11 @@ pub unsafe fn warp_shuffle_idx<T: WarpShuffleValue>(
 ///
 /// - `mask` dictates what threads will participate in the shuffle, usually [`u32::MAX`] to indicate all threads.
 /// - `value` is the value that will be shuffled across the threads. i.e. the value that will be given to the thread
-/// that calculates this thread as its target lane.
+///   that calculates this thread as its target lane.
 /// - `lane_mask` is the value that will be XOR'd by the current thread's lane id to calculate the target lane. i.e. the
-/// target lane will be `lane_id ^ lane_mask`.
+///   target lane will be `lane_id ^ lane_mask`.
 /// - `width` dictates how to optionally split the warp into subsections, it must be a power of two and lower than `32`.
-/// calculated source lane values will NOT wrap around the value of `width`. Usually just `32`.
+///   calculated source lane values will NOT wrap around the value of `width`. Usually just `32`.
 ///
 /// # Returns
 ///
@@ -565,7 +567,7 @@ pub unsafe fn warp_shuffle_idx<T: WarpShuffleValue>(
 /// Behavior is undefined if:
 /// - Any thread participating in the shuffle has exited or the executing thread is not in `mask`.
 /// - For `compute_62` and below, all threads in `mask` must call the same function in convergence, and only the threads
-/// in `mask` can be active when the shuffle is called.
+///   in `mask` can be active when the shuffle is called.
 ///
 /// The returned value returned is unspecified if the calculated target lane is inactive.
 pub unsafe fn warp_shuffle_xor<T: WarpShuffleValue>(
diff --git a/crates/cuda_std_macros/src/lib.rs b/crates/cuda_std_macros/src/lib.rs
index 1d31dbaa..149346ef 100644
--- a/crates/cuda_std_macros/src/lib.rs
+++ b/crates/cuda_std_macros/src/lib.rs
@@ -27,7 +27,7 @@ pub fn kernel(input: proc_macro::TokenStream, item: proc_macro::TokenStream) ->
     let mut item = parse_macro_input!(item as ItemFn);
     let no_mangle = parse_quote!(#[no_mangle]);
     item.attrs.push(no_mangle);
-    let internal = parse_quote!(#[cfg_attr(target_arch="nvptx64", nvvm_internal(kernel(#input)))]);
+    let internal = parse_quote!(#[cfg_attr(target_arch="nvptx64", nvvm_internal::kernel(#input))]);
     item.attrs.push(internal);
 
     // used to guarantee some things about how params are passed in the codegen.
@@ -155,12 +155,7 @@ pub fn gpu_only(_attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -
     } = syn::parse_macro_input!(item as syn::ItemFn);
 
     let mut cloned_attrs = attrs.clone();
-    cloned_attrs.retain(|a| {
-        !a.path()
-            .get_ident()
-            .map(|x| *x == "nvvm_internal")
-            .unwrap_or_default()
-    });
+    cloned_attrs.retain(|a| a.path().segments[0].ident != "nvvm_internal");
 
     let fn_name = sig.ident.clone();
 
@@ -203,7 +198,7 @@ pub fn externally_visible(
         "#[externally_visible] function should also be #[no_mangle]"
     );
 
-    let new_attr = parse_quote!(#[cfg_attr(target_os = "cuda", nvvm_internal(used))]);
+    let new_attr = parse_quote!(#[cfg_attr(target_os = "cuda", nvvm_internal::used)]);
     func.attrs.push(new_attr);
 
     func.into_token_stream().into()
@@ -231,7 +226,7 @@ pub fn address_space(attr: proc_macro::TokenStream, item: proc_macro::TokenStrea
     };
 
     let new_attr =
-        parse_quote!(#[cfg_attr(target_os = "cuda", nvvm_internal(addrspace(#addrspace_num)))]);
+        parse_quote!(#[cfg_attr(target_os = "cuda", nvvm_internal::addrspace(#addrspace_num))]);
     global.attrs.push(new_attr);
 
     global.into_token_stream().into()
diff --git a/crates/cust/Cargo.toml b/crates/cust/Cargo.toml
index bb5a3427..5a609176 100644
--- a/crates/cust/Cargo.toml
+++ b/crates/cust/Cargo.toml
@@ -35,7 +35,7 @@ impl_num_complex = ["cust_core/num-complex", "num-complex"]
 find_cuda_helper = { path = "../find_cuda_helper", version = "0.2" }
 
 [dev-dependencies]
-image = "0.23.14"
+image = "0.25.5"
 
 [package.metadata.docs.rs]
 rustdoc-args = ["--cfg", "docsrs"]
diff --git a/crates/cust_core/src/lib.rs b/crates/cust_core/src/lib.rs
index ed4ddd35..c307932d 100644
--- a/crates/cust_core/src/lib.rs
+++ b/crates/cust_core/src/lib.rs
@@ -93,7 +93,7 @@ pub mod _hidden {
     unsafe impl<T: DeviceCopy> DeviceCopy for MaybeUninit<T> {}
     unsafe impl<T: DeviceCopy> DeviceCopy for Option<T> {}
     unsafe impl<L: DeviceCopy, R: DeviceCopy> DeviceCopy for Result<L, R> {}
-    unsafe impl<T: ?Sized + DeviceCopy> DeviceCopy for PhantomData<T> {}
+    unsafe impl<T: DeviceCopy> DeviceCopy for PhantomData<T> {}
     // Allow DeviceCopy for lifetime constraint markers
     unsafe impl DeviceCopy for PhantomData<&()> {}
     unsafe impl<T: DeviceCopy> DeviceCopy for Wrapping<T> {}
diff --git a/crates/find_cuda_helper/src/lib.rs b/crates/find_cuda_helper/src/lib.rs
index 52d281f7..ed3b2f46 100644
--- a/crates/find_cuda_helper/src/lib.rs
+++ b/crates/find_cuda_helper/src/lib.rs
@@ -1,6 +1,5 @@
 //! Tiny crate for common logic for finding and including CUDA.
 
-use std::process::Command;
 use std::{
     env,
     path::{Path, PathBuf},
@@ -154,6 +153,7 @@ pub fn find_cuda_lib_dirs() -> Vec<PathBuf> {
 
 #[cfg(not(target_os = "windows"))]
 fn detect_cuda_root_via_which_nvcc() -> PathBuf {
+    use std::process::Command;
     let output = Command::new("which")
         .arg("nvcc")
         .output()
diff --git a/crates/optix/Cargo.toml b/crates/optix/Cargo.toml
index 815e4bbe..61f2fe32 100644
--- a/crates/optix/Cargo.toml
+++ b/crates/optix/Cargo.toml
@@ -27,7 +27,7 @@ mint = "0.5.9"
 embed-doc-image = {version = "0.1.4"}
 
 [build-dependencies]
-bindgen = "0.59"
+bindgen = "0.71.1"
 cc = "1.0.71"
 find_cuda_helper = { version = "0.2", path = "../find_cuda_helper" }
 
diff --git a/crates/optix/build.rs b/crates/optix/build.rs
index c48dcdd2..1356645d 100644
--- a/crates/optix/build.rs
+++ b/crates/optix/build.rs
@@ -83,10 +83,10 @@ fn bindgen_optix(optix_include: &Path, cuda_include: &Path) {
         .rustified_enum("OptixGeometryFlags")
         .constified_enum("OptixVertexFormat")
         .constified_enum("OptixIndicesFormat")
-        .rust_target(bindgen::RustTarget::Nightly)
+        .rust_target(bindgen::RustTarget::nightly())
         .derive_default(true)
         .derive_partialeq(true)
-        .rustfmt_bindings(true)
+        .formatter(bindgen::Formatter::Rustfmt)
         .generate()
         .expect("Unable to generate optix bindings");
 
diff --git a/crates/optix/examples/ex02_pipeline/device/src/lib.rs b/crates/optix/examples/ex02_pipeline/device/src/lib.rs
index 1654a509..b2027dfe 100644
--- a/crates/optix/examples/ex02_pipeline/device/src/lib.rs
+++ b/crates/optix/examples/ex02_pipeline/device/src/lib.rs
@@ -1,9 +1,5 @@
-#![cfg_attr(
-    target_os = "cuda",
-    no_std,
-    feature(register_attr),
-    register_attr(nvvm_internal)
-)]
+#![feature(asm)]
+#![cfg_attr(target_os = "cuda", no_std, register_attr(nvvm_internal))]
 // #![deny(warnings)]
 #![allow(clippy::missing_safety_doc)]
 
diff --git a/crates/optix/examples/rust/ex04_mesh_gpu/src/lib.rs b/crates/optix/examples/rust/ex04_mesh_gpu/src/lib.rs
index 3266975e..cd12a24b 100644
--- a/crates/optix/examples/rust/ex04_mesh_gpu/src/lib.rs
+++ b/crates/optix/examples/rust/ex04_mesh_gpu/src/lib.rs
@@ -1,9 +1,4 @@
-#![cfg_attr(
-    target_os = "cuda",
-    no_std,
-    feature(register_attr),
-    register_attr(nvvm_internal)
-)]
+#![cfg_attr(target_os = "cuda", no_std, register_attr(nvvm_internal))]
 #![allow(non_snake_case, clippy::missing_safety_doc)]
 
 use cuda_std::kernel;
diff --git a/crates/optix/optix_wrapper.rs b/crates/optix/optix_wrapper.rs
index 24bad9e9..7f9740db 100644
--- a/crates/optix/optix_wrapper.rs
+++ b/crates/optix/optix_wrapper.rs
@@ -1,4 +1,4 @@
-/* automatically generated by rust-bindgen 0.59.2 */
+/* automatically generated by rust-bindgen 0.71.1 */
 
 #[repr(C)]
 pub struct __BindgenUnionField<T>(::std::marker::PhantomData<T>);
@@ -25,7 +25,7 @@ impl<T> ::std::default::Default for __BindgenUnionField<T> {
 impl<T> ::std::clone::Clone for __BindgenUnionField<T> {
     #[inline]
     fn clone(&self) -> Self {
-        Self::new()
+        *self
     }
 }
 impl<T> ::std::marker::Copy for __BindgenUnionField<T> {}
@@ -447,19 +447,11 @@ impl Default for OptixAccelBuildOptions {
     }
 }
 #[repr(C)]
+#[derive(Debug, Default, Copy, Clone, PartialEq)]
 pub struct OptixAccelBufferSizes {
-    pub outputSizeInBytes: size_t,
-    pub tempSizeInBytes: size_t,
-    pub tempUpdateSizeInBytes: size_t,
-}
-impl Default for OptixAccelBufferSizes {
-    fn default() -> Self {
-        let mut s = ::std::mem::MaybeUninit::<Self>::uninit();
-        unsafe {
-            ::std::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
-            s.assume_init()
-        }
-    }
+    pub outputSizeInBytes: usize,
+    pub tempSizeInBytes: usize,
+    pub tempUpdateSizeInBytes: usize,
 }
 pub const OptixAccelPropertyType_OPTIX_PROPERTY_TYPE_COMPACTED_SIZE: OptixAccelPropertyType = 8577;
 pub const OptixAccelPropertyType_OPTIX_PROPERTY_TYPE_AABBS: OptixAccelPropertyType = 8578;
@@ -622,21 +614,13 @@ impl Default for OptixDenoiserParams {
     }
 }
 #[repr(C)]
+#[derive(Debug, Default, Copy, Clone, PartialEq)]
 pub struct OptixDenoiserSizes {
-    pub stateSizeInBytes: size_t,
-    pub withOverlapScratchSizeInBytes: size_t,
-    pub withoutOverlapScratchSizeInBytes: size_t,
+    pub stateSizeInBytes: usize,
+    pub withOverlapScratchSizeInBytes: usize,
+    pub withoutOverlapScratchSizeInBytes: usize,
     pub overlapWindowSizeInPixels: ::std::os::raw::c_uint,
 }
-impl Default for OptixDenoiserSizes {
-    fn default() -> Self {
-        let mut s = ::std::mem::MaybeUninit::<Self>::uninit();
-        unsafe {
-            ::std::ptr::write_bytes(s.as_mut_ptr(), 0, 1);
-            s.assume_init()
-        }
-    }
-}
 pub const OptixRayFlags_OPTIX_RAY_FLAG_NONE: OptixRayFlags = 0;
 pub const OptixRayFlags_OPTIX_RAY_FLAG_DISABLE_ANYHIT: OptixRayFlags = 1;
 pub const OptixRayFlags_OPTIX_RAY_FLAG_ENFORCE_ANYHIT: OptixRayFlags = 2;
@@ -675,9 +659,10 @@ pub mod OptixCompileDebugLevel {
     pub const OPTIX_COMPILE_DEBUG_LEVEL_FULL: Type = 9042;
 }
 #[repr(C)]
+#[derive(Debug, Copy, Clone, PartialEq)]
 pub struct OptixModuleCompileBoundValueEntry {
-    pub pipelineParamOffsetInBytes: size_t,
-    pub sizeInBytes: size_t,
+    pub pipelineParamOffsetInBytes: usize,
+    pub sizeInBytes: usize,
     pub boundValuePtr: *const ::std::os::raw::c_void,
     pub annotation: *const ::std::os::raw::c_char,
 }
@@ -849,6 +834,7 @@ pub mod OptixExceptionFlags {
     pub const OPTIX_EXCEPTION_FLAG_DEBUG: Type = 8;
 }
 #[repr(C)]
+#[derive(Debug, Copy, Clone, PartialEq)]
 pub struct OptixPipelineCompileOptions {
     pub usesMotionBlur: ::std::os::raw::c_int,
     pub traversableGraphFlags: ::std::os::raw::c_uint,
@@ -858,7 +844,7 @@ pub struct OptixPipelineCompileOptions {
     pub pipelineLaunchParamsVariableName: *const ::std::os::raw::c_char,
     pub usesPrimitiveTypeFlags: ::std::os::raw::c_uint,
     pub reserved: ::std::os::raw::c_uint,
-    pub reserved2: size_t,
+    pub reserved2: usize,
 }
 impl Default for OptixPipelineCompileOptions {
     fn default() -> Self {
@@ -928,7 +914,7 @@ pub type OptixQueryFunctionTable_t = ::std::option::Option<
         arg1: *mut OptixQueryFunctionTableOptions,
         arg2: *mut *const ::std::os::raw::c_void,
         functionTable: *mut ::std::os::raw::c_void,
-        sizeOfTable: size_t,
+        sizeOfTable: usize,
     ) -> OptixResult,
 >;
 #[repr(C)]
@@ -946,31 +932,31 @@ impl Default for OptixBuiltinISOptions {
         }
     }
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixGetErrorName(result: OptixResult) -> *const ::std::os::raw::c_char;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixGetErrorString(result: OptixResult) -> *const ::std::os::raw::c_char;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextCreate(
         fromContext: CUcontext,
         options: *const OptixDeviceContextOptions,
         context: *mut OptixDeviceContext,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextDestroy(context: OptixDeviceContext) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextGetProperty(
         context: OptixDeviceContext,
         property: OptixDeviceProperty::Type,
         value: *mut ::std::os::raw::c_void,
-        sizeInBytes: size_t,
+        sizeInBytes: usize,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextSetLogCallback(
         context: OptixDeviceContext,
         callbackFunction: OptixLogCallback,
@@ -978,46 +964,46 @@ extern "C" {
         callbackLevel: ::std::os::raw::c_uint,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextSetCacheEnabled(
         context: OptixDeviceContext,
         enabled: ::std::os::raw::c_int,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextSetCacheLocation(
         context: OptixDeviceContext,
         location: *const ::std::os::raw::c_char,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextSetCacheDatabaseSizes(
         context: OptixDeviceContext,
-        lowWaterMark: size_t,
-        highWaterMark: size_t,
+        lowWaterMark: usize,
+        highWaterMark: usize,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextGetCacheEnabled(
         context: OptixDeviceContext,
         enabled: *mut ::std::os::raw::c_int,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextGetCacheLocation(
         context: OptixDeviceContext,
         location: *mut ::std::os::raw::c_char,
-        locationSize: size_t,
+        locationSize: usize,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDeviceContextGetCacheDatabaseSizes(
         context: OptixDeviceContext,
-        lowWaterMark: *mut size_t,
-        highWaterMark: *mut size_t,
+        lowWaterMark: *mut usize,
+        highWaterMark: *mut usize,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixPipelineCreate(
         context: OptixDeviceContext,
         pipelineCompileOptions: *const OptixPipelineCompileOptions,
@@ -1025,14 +1011,14 @@ extern "C" {
         programGroups: *const OptixProgramGroup,
         numProgramGroups: ::std::os::raw::c_uint,
         logString: *mut ::std::os::raw::c_char,
-        logStringSize: *mut size_t,
+        logStringSize: *mut usize,
         pipeline: *mut OptixPipeline,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixPipelineDestroy(pipeline: OptixPipeline) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixPipelineSetStackSize(
         pipeline: OptixPipeline,
         directCallableStackSizeFromTraversal: ::std::os::raw::c_uint,
@@ -1041,22 +1027,22 @@ extern "C" {
         maxTraversableGraphDepth: ::std::os::raw::c_uint,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixModuleCreateFromPTX(
         context: OptixDeviceContext,
         moduleCompileOptions: *const OptixModuleCompileOptions,
         pipelineCompileOptions: *const OptixPipelineCompileOptions,
         PTX: *const ::std::os::raw::c_char,
-        PTXsize: size_t,
+        PTXsize: usize,
         logString: *mut ::std::os::raw::c_char,
-        logStringSize: *mut size_t,
+        logStringSize: *mut usize,
         module: *mut OptixModule,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixModuleDestroy(module: OptixModule) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixBuiltinISModuleGet(
         context: OptixDeviceContext,
         moduleCompileOptions: *const OptixModuleCompileOptions,
@@ -1065,45 +1051,45 @@ extern "C" {
         builtinModule: *mut OptixModule,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixProgramGroupGetStackSize(
         programGroup: OptixProgramGroup,
         stackSizes: *mut OptixStackSizes,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixProgramGroupCreate(
         context: OptixDeviceContext,
         programDescriptions: *const OptixProgramGroupDesc,
         numProgramGroups: ::std::os::raw::c_uint,
         options: *const OptixProgramGroupOptions,
         logString: *mut ::std::os::raw::c_char,
-        logStringSize: *mut size_t,
+        logStringSize: *mut usize,
         programGroups: *mut OptixProgramGroup,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixProgramGroupDestroy(programGroup: OptixProgramGroup) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixLaunch(
         pipeline: OptixPipeline,
         stream: CUstream,
         pipelineParams: CUdeviceptr,
-        pipelineParamsSize: size_t,
+        pipelineParamsSize: usize,
         sbt: *const OptixShaderBindingTable,
         width: ::std::os::raw::c_uint,
         height: ::std::os::raw::c_uint,
         depth: ::std::os::raw::c_uint,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixSbtRecordPackHeader(
         programGroup: OptixProgramGroup,
         sbtRecordHeaderHostPointer: *mut ::std::os::raw::c_void,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixAccelComputeMemoryUsage(
         context: OptixDeviceContext,
         accelOptions: *const OptixAccelBuildOptions,
@@ -1112,7 +1098,7 @@ extern "C" {
         bufferSizes: *mut OptixAccelBufferSizes,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixAccelBuild(
         context: OptixDeviceContext,
         stream: CUstream,
@@ -1120,51 +1106,51 @@ extern "C" {
         buildInputs: *const OptixBuildInput,
         numBuildInputs: ::std::os::raw::c_uint,
         tempBuffer: CUdeviceptr,
-        tempBufferSizeInBytes: size_t,
+        tempBufferSizeInBytes: usize,
         outputBuffer: CUdeviceptr,
-        outputBufferSizeInBytes: size_t,
+        outputBufferSizeInBytes: usize,
         outputHandle: *mut OptixTraversableHandle,
         emittedProperties: *const OptixAccelEmitDesc,
         numEmittedProperties: ::std::os::raw::c_uint,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixAccelGetRelocationInfo(
         context: OptixDeviceContext,
         handle: OptixTraversableHandle,
         info: *mut OptixAccelRelocationInfo,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixAccelCheckRelocationCompatibility(
         context: OptixDeviceContext,
         info: *const OptixAccelRelocationInfo,
         compatible: *mut ::std::os::raw::c_int,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixAccelRelocate(
         context: OptixDeviceContext,
         stream: CUstream,
         info: *const OptixAccelRelocationInfo,
         instanceTraversableHandles: CUdeviceptr,
-        numInstanceTraversableHandles: size_t,
+        numInstanceTraversableHandles: usize,
         targetAccel: CUdeviceptr,
-        targetAccelSizeInBytes: size_t,
+        targetAccelSizeInBytes: usize,
         targetHandle: *mut OptixTraversableHandle,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixAccelCompact(
         context: OptixDeviceContext,
         stream: CUstream,
         inputHandle: OptixTraversableHandle,
         outputBuffer: CUdeviceptr,
-        outputBufferSizeInBytes: size_t,
+        outputBufferSizeInBytes: usize,
         outputHandle: *mut OptixTraversableHandle,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixConvertPointerToTraversableHandle(
         onDevice: OptixDeviceContext,
         pointer: CUdeviceptr,
@@ -1172,7 +1158,7 @@ extern "C" {
         traversableHandle: *mut OptixTraversableHandle,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDenoiserCreate(
         context: OptixDeviceContext,
         modelKind: OptixDenoiserModelKind::Type,
@@ -1180,18 +1166,18 @@ extern "C" {
         denoiser: *mut OptixDenoiser,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDenoiserCreateWithUserModel(
         context: OptixDeviceContext,
         userData: *const ::std::os::raw::c_void,
-        userDataSizeInBytes: size_t,
+        userDataSizeInBytes: usize,
         denoiser: *mut OptixDenoiser,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDenoiserDestroy(denoiser: OptixDenoiser) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDenoiserComputeMemoryResources(
         denoiser: OptixDenoiser,
         outputWidth: ::std::os::raw::c_uint,
@@ -1199,52 +1185,52 @@ extern "C" {
         returnSizes: *mut OptixDenoiserSizes,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDenoiserSetup(
         denoiser: OptixDenoiser,
         stream: CUstream,
         inputWidth: ::std::os::raw::c_uint,
         inputHeight: ::std::os::raw::c_uint,
         denoiserState: CUdeviceptr,
-        denoiserStateSizeInBytes: size_t,
+        denoiserStateSizeInBytes: usize,
         scratch: CUdeviceptr,
-        scratchSizeInBytes: size_t,
+        scratchSizeInBytes: usize,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDenoiserInvoke(
         denoiser: OptixDenoiser,
         stream: CUstream,
         params: *const OptixDenoiserParams,
         denoiserState: CUdeviceptr,
-        denoiserStateSizeInBytes: size_t,
+        denoiserStateSizeInBytes: usize,
         guideLayer: *const OptixDenoiserGuideLayer,
         layers: *const OptixDenoiserLayer,
         numLayers: ::std::os::raw::c_uint,
         inputOffsetX: ::std::os::raw::c_uint,
         inputOffsetY: ::std::os::raw::c_uint,
         scratch: CUdeviceptr,
-        scratchSizeInBytes: size_t,
+        scratchSizeInBytes: usize,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDenoiserComputeIntensity(
         denoiser: OptixDenoiser,
         stream: CUstream,
         inputImage: *const OptixImage2D,
         outputIntensity: CUdeviceptr,
         scratch: CUdeviceptr,
-        scratchSizeInBytes: size_t,
+        scratchSizeInBytes: usize,
     ) -> OptixResult;
 }
-extern "C" {
+unsafe extern "C" {
     pub fn optixDenoiserComputeAverageColor(
         denoiser: OptixDenoiser,
         stream: CUstream,
         inputImage: *const OptixImage2D,
         outputAverageColor: CUdeviceptr,
         scratch: CUdeviceptr,
-        scratchSizeInBytes: size_t,
+        scratchSizeInBytes: usize,
     ) -> OptixResult;
 }
 #[repr(C)]
@@ -1270,7 +1256,7 @@ pub struct OptixFunctionTable {
             context: OptixDeviceContext,
             property: OptixDeviceProperty::Type,
             value: *mut ::std::os::raw::c_void,
-            sizeInBytes: size_t,
+            sizeInBytes: usize,
         ) -> OptixResult,
     >,
     pub optixDeviceContextSetLogCallback: ::std::option::Option<
@@ -1296,8 +1282,8 @@ pub struct OptixFunctionTable {
     pub optixDeviceContextSetCacheDatabaseSizes: ::std::option::Option<
         unsafe extern "C" fn(
             context: OptixDeviceContext,
-            lowWaterMark: size_t,
-            highWaterMark: size_t,
+            lowWaterMark: usize,
+            highWaterMark: usize,
         ) -> OptixResult,
     >,
     pub optixDeviceContextGetCacheEnabled: ::std::option::Option<
@@ -1310,14 +1296,14 @@ pub struct OptixFunctionTable {
         unsafe extern "C" fn(
             context: OptixDeviceContext,
             location: *mut ::std::os::raw::c_char,
-            locationSize: size_t,
+            locationSize: usize,
         ) -> OptixResult,
     >,
     pub optixDeviceContextGetCacheDatabaseSizes: ::std::option::Option<
         unsafe extern "C" fn(
             context: OptixDeviceContext,
-            lowWaterMark: *mut size_t,
-            highWaterMark: *mut size_t,
+            lowWaterMark: *mut usize,
+            highWaterMark: *mut usize,
         ) -> OptixResult,
     >,
     pub optixModuleCreateFromPTX: ::std::option::Option<
@@ -1326,9 +1312,9 @@ pub struct OptixFunctionTable {
             moduleCompileOptions: *const OptixModuleCompileOptions,
             pipelineCompileOptions: *const OptixPipelineCompileOptions,
             PTX: *const ::std::os::raw::c_char,
-            PTXsize: size_t,
+            PTXsize: usize,
             logString: *mut ::std::os::raw::c_char,
-            logStringSize: *mut size_t,
+            logStringSize: *mut usize,
             module: *mut OptixModule,
         ) -> OptixResult,
     >,
@@ -1350,7 +1336,7 @@ pub struct OptixFunctionTable {
             numProgramGroups: ::std::os::raw::c_uint,
             options: *const OptixProgramGroupOptions,
             logString: *mut ::std::os::raw::c_char,
-            logStringSize: *mut size_t,
+            logStringSize: *mut usize,
             programGroups: *mut OptixProgramGroup,
         ) -> OptixResult,
     >,
@@ -1370,7 +1356,7 @@ pub struct OptixFunctionTable {
             programGroups: *const OptixProgramGroup,
             numProgramGroups: ::std::os::raw::c_uint,
             logString: *mut ::std::os::raw::c_char,
-            logStringSize: *mut size_t,
+            logStringSize: *mut usize,
             pipeline: *mut OptixPipeline,
         ) -> OptixResult,
     >,
@@ -1402,9 +1388,9 @@ pub struct OptixFunctionTable {
             buildInputs: *const OptixBuildInput,
             numBuildInputs: ::std::os::raw::c_uint,
             tempBuffer: CUdeviceptr,
-            tempBufferSizeInBytes: size_t,
+            tempBufferSizeInBytes: usize,
             outputBuffer: CUdeviceptr,
-            outputBufferSizeInBytes: size_t,
+            outputBufferSizeInBytes: usize,
             outputHandle: *mut OptixTraversableHandle,
             emittedProperties: *const OptixAccelEmitDesc,
             numEmittedProperties: ::std::os::raw::c_uint,
@@ -1430,9 +1416,9 @@ pub struct OptixFunctionTable {
             stream: CUstream,
             info: *const OptixAccelRelocationInfo,
             instanceTraversableHandles: CUdeviceptr,
-            numInstanceTraversableHandles: size_t,
+            numInstanceTraversableHandles: usize,
             targetAccel: CUdeviceptr,
-            targetAccelSizeInBytes: size_t,
+            targetAccelSizeInBytes: usize,
             targetHandle: *mut OptixTraversableHandle,
         ) -> OptixResult,
     >,
@@ -1442,7 +1428,7 @@ pub struct OptixFunctionTable {
             stream: CUstream,
             inputHandle: OptixTraversableHandle,
             outputBuffer: CUdeviceptr,
-            outputBufferSizeInBytes: size_t,
+            outputBufferSizeInBytes: usize,
             outputHandle: *mut OptixTraversableHandle,
         ) -> OptixResult,
     >,
@@ -1465,7 +1451,7 @@ pub struct OptixFunctionTable {
             pipeline: OptixPipeline,
             stream: CUstream,
             pipelineParams: CUdeviceptr,
-            pipelineParamsSize: size_t,
+            pipelineParamsSize: usize,
             sbt: *const OptixShaderBindingTable,
             width: ::std::os::raw::c_uint,
             height: ::std::os::raw::c_uint,
@@ -1497,9 +1483,9 @@ pub struct OptixFunctionTable {
             inputWidth: ::std::os::raw::c_uint,
             inputHeight: ::std::os::raw::c_uint,
             state: CUdeviceptr,
-            stateSizeInBytes: size_t,
+            stateSizeInBytes: usize,
             scratch: CUdeviceptr,
-            scratchSizeInBytes: size_t,
+            scratchSizeInBytes: usize,
         ) -> OptixResult,
     >,
     pub optixDenoiserInvoke: ::std::option::Option<
@@ -1508,14 +1494,14 @@ pub struct OptixFunctionTable {
             stream: CUstream,
             params: *const OptixDenoiserParams,
             denoiserState: CUdeviceptr,
-            denoiserStateSizeInBytes: size_t,
+            denoiserStateSizeInBytes: usize,
             guideLayer: *const OptixDenoiserGuideLayer,
             layers: *const OptixDenoiserLayer,
             numLayers: ::std::os::raw::c_uint,
             inputOffsetX: ::std::os::raw::c_uint,
             inputOffsetY: ::std::os::raw::c_uint,
             scratch: CUdeviceptr,
-            scratchSizeInBytes: size_t,
+            scratchSizeInBytes: usize,
         ) -> OptixResult,
     >,
     pub optixDenoiserComputeIntensity: ::std::option::Option<
@@ -1525,7 +1511,7 @@ pub struct OptixFunctionTable {
             inputImage: *const OptixImage2D,
             outputIntensity: CUdeviceptr,
             scratch: CUdeviceptr,
-            scratchSizeInBytes: size_t,
+            scratchSizeInBytes: usize,
         ) -> OptixResult,
     >,
     pub optixDenoiserComputeAverageColor: ::std::option::Option<
@@ -1535,28 +1521,28 @@ pub struct OptixFunctionTable {
             inputImage: *const OptixImage2D,
             outputAverageColor: CUdeviceptr,
             scratch: CUdeviceptr,
-            scratchSizeInBytes: size_t,
+            scratchSizeInBytes: usize,
         ) -> OptixResult,
     >,
     pub optixDenoiserCreateWithUserModel: ::std::option::Option<
         unsafe extern "C" fn(
             context: OptixDeviceContext,
             data: *const ::std::os::raw::c_void,
-            dataSizeInBytes: size_t,
+            dataSizeInBytes: usize,
             returnHandle: *mut OptixDenoiser,
         ) -> OptixResult,
     >,
 }
-pub const OptixSbtRecordHeaderSize: size_t = 32;
-pub const OptixSbtRecordAlignment: size_t = 16;
-pub const OptixAccelBufferByteAlignment: size_t = 128;
-pub const OptixInstanceByteAlignment: size_t = 16;
-pub const OptixAabbBufferByteAlignment: size_t = 8;
-pub const OptixGeometryTransformByteAlignment: size_t = 16;
-pub const OptixTransformByteAlignment: size_t = 64;
-pub const OptixVersion: size_t = 70300;
-pub const OptixBuildInputSize: size_t = 1032;
-pub const OptixShaderBindingTableSize: size_t = 64;
+pub const OptixSbtRecordHeaderSize: usize = 32;
+pub const OptixSbtRecordAlignment: usize = 16;
+pub const OptixAccelBufferByteAlignment: usize = 128;
+pub const OptixInstanceByteAlignment: usize = 16;
+pub const OptixAabbBufferByteAlignment: usize = 8;
+pub const OptixGeometryTransformByteAlignment: usize = 16;
+pub const OptixTransformByteAlignment: usize = 64;
+pub const OptixVersion: usize = 70300;
+pub const OptixBuildInputSize: usize = 1032;
+pub const OptixShaderBindingTableSize: usize = 64;
 #[repr(i32)]
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub enum OptixGeometryFlags {
diff --git a/crates/optix_device/src/lib.rs b/crates/optix_device/src/lib.rs
index d75938be..1b54795c 100644
--- a/crates/optix_device/src/lib.rs
+++ b/crates/optix_device/src/lib.rs
@@ -1,9 +1,10 @@
 #![cfg_attr(
     target_arch = "nvptx64",
     no_std,
-    feature(register_attr, asm, asm_experimental_arch),
+    feature(asm_experimental_arch),
     register_attr(nvvm_internal)
 )]
+use core::arch::asm;
 
 extern crate alloc;
 
diff --git a/crates/rustc_codegen_nvvm/Cargo.toml b/crates/rustc_codegen_nvvm/Cargo.toml
index acdc91a8..054a0927 100644
--- a/crates/rustc_codegen_nvvm/Cargo.toml
+++ b/crates/rustc_codegen_nvvm/Cargo.toml
@@ -5,7 +5,7 @@ authors = [
     "Riccardo D'Ambrosio <rdambrosio016@gmail.com>",
     "The Rust Project Developers",
 ]
-edition = "2018"
+edition = "2024"
 license = "MIT OR Apache-2.0"
 description = "A codegen backend for Rustc which targets the libnvvm CUDA library"
 repository = "https://github.com/Rust-GPU/Rust-CUDA"
@@ -18,12 +18,19 @@ crate-type = ["dylib"]
 nvvm = { version = "0.1", path = "../nvvm" }
 rustc-demangle = "0.1.24"
 libc = "0.2.169"
+libloading = "0.8.0"
 tar = "0.4.43"
-bitflags = "2.8"
+object = "0.36.7"
+bitflags = "2.8.0"
+# To avoid duplicate dependencies, this should match the version of gimli used
+# by `rustc_codegen_ssa` via its `thorin-dwp` dependency.
+gimli = "0.30"
 tracing = { version = "0.1.41", features = ["release_max_level_debug"] }
 find_cuda_helper = { version = "0.2", path = "../find_cuda_helper" }
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
 rustc_codegen_nvvm_macros = { version = "0.1", path = "../rustc_codegen_nvvm_macros" }
+smallvec = { version = "1.14.0", features = ["union", "may_dangle"] }
+itertools = "0.14.0"
 
 [build-dependencies]
 build-helper = "0.1.1"
diff --git a/crates/rustc_codegen_nvvm/build.rs b/crates/rustc_codegen_nvvm/build.rs
index 2b663329..d1879c09 100644
--- a/crates/rustc_codegen_nvvm/build.rs
+++ b/crates/rustc_codegen_nvvm/build.rs
@@ -55,7 +55,10 @@ fn target_to_llvm_prebuilt(target: &str) -> String {
         "x86_64-pc-windows-msvc" => "windows-x86_64",
         // NOTE(RDambrosio016): currently disabled because of weird issues with segfaults and building the C++ shim
         // "x86_64-unknown-linux-gnu" => "linux-x86_64",
-        _ => panic!("Unsupported target with no matching prebuilt LLVM: `{}`, install LLVM and set LLVM_CONFIG", target)
+        _ => panic!(
+            "Unsupported target with no matching prebuilt LLVM: `{}`, install LLVM and set LLVM_CONFIG",
+            target
+        ),
     };
     format!("{}.tar.xz", base)
 }
@@ -97,7 +100,7 @@ fn find_llvm_config(target: &str) -> PathBuf {
     let mut easy = Easy::new();
 
     easy.url(&url).unwrap();
-    let _redirect = easy.follow_location(true).unwrap();
+    easy.follow_location(true).unwrap();
     let mut xz_encoded = Vec::with_capacity(20_000_000); // 20mb
     {
         let mut transfer = easy.transfer();
diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp
index ee3baabd..5c0f5e83 100644
--- a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp
+++ b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp
@@ -96,9 +96,9 @@ static LLVMRustPassKind toRust(PassKind Kind)
   }
 }
 
-extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName)
+extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName, size_t PassNameLen)
 {
-  StringRef SR(PassName);
+  StringRef SR(PassName, PassNameLen);
   PassRegistry *PR = PassRegistry::getPassRegistry();
 
   const PassInfo *PI = PR->getPassInfo(SR);
@@ -377,7 +377,9 @@ extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef)
 #endif
 
 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
-    const char *TripleStr, const char *CPU, const char *Feature,
+    const char *TripleStr, size_t TripleStrLen,
+    const char *CPU, size_t CPULen,
+    const char *Feature, size_t FeatureLen,
     LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
     LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
     bool PositionIndependentExecutable, bool FunctionSections,
@@ -390,7 +392,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   auto RM = fromRust(RustReloc);
 
   std::string Error;
-  Triple Trip(Triple::normalize(TripleStr));
+  Triple Trip(Triple::normalize(StringRef(TripleStr, TripleStrLen)));
   const llvm::Target *TheTarget =
       TargetRegistry::lookupTarget(Trip.getTriple(), Error);
   if (TheTarget == nullptr)
@@ -399,7 +401,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     return nullptr;
   }
 
-  StringRef RealCPU = CPU;
+  StringRef RealCPU{CPU, CPULen};
   if (RealCPU == "native")
   {
     RealCPU = sys::getHostCPUName();
@@ -437,7 +439,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   if (RustCM != LLVMRustCodeModel::None)
     CM = fromRust(RustCM);
   TargetMachine *TM = TheTarget->createTargetMachine(
-      Trip.getTriple(), RealCPU, Feature, Options, RM, CM, OptLevel);
+      Trip.getTriple(), RealCPU, StringRef(Feature, FeatureLen), Options, RM, CM, OptLevel);
   return wrap(TM);
 }
 
@@ -763,11 +765,11 @@ INITIALIZE_PASS(RustPrintModulePass, "print-rust-module",
 // }
 
 extern "C" LLVMRustResult
-LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle)
+LLVMRustPrintModule(LLVMModuleRef M, const char *Path, size_t PathLen, DemangleFn Demangle)
 {
   std::string ErrorInfo;
   std::error_code EC;
-  raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
+  raw_fd_ostream OS(StringRef(Path, PathLen), EC, sys::fs::OF_None);
   if (EC)
     ErrorInfo = EC.message();
   if (ErrorInfo != "")
@@ -936,11 +938,12 @@ LLVMRustPGOAvailable()
 extern "C" bool
 LLVMRustWriteThinBitcodeToFile(LLVMPassManagerRef PMR,
                                LLVMModuleRef M,
-                               const char *BcFile)
+                               const char *BcFile,
+                               size_t BcFileLen)
 {
   llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
   std::error_code EC;
-  llvm::raw_fd_ostream bc(BcFile, EC, llvm::sys::fs::F_None);
+  llvm::raw_fd_ostream bc(StringRef(BcFile, BcFileLen), EC, llvm::sys::fs::F_None);
   if (EC)
   {
     LLVMRustSetLastError(EC.message().c_str());
@@ -1285,10 +1288,11 @@ extern "C" LLVMModuleRef
 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
                            const char *data,
                            size_t len,
-                           const char *identifier)
+                           const char *identifier,
+                           size_t idLen)
 {
   StringRef Data(data, len);
-  MemoryBufferRef Buffer(Data, identifier);
+  MemoryBufferRef Buffer{Data, StringRef(identifier, idLen)};
   unwrap(Context)->enableDebugTypeODRUniquing();
   Expected<std::unique_ptr<Module>> SrcOrError =
       parseBitcodeFile(Buffer, *unwrap(Context));
@@ -1393,7 +1397,8 @@ LLVMRustThinLTORemoveAvailableExternally(LLVMModuleRef Mod)
 extern "C" bool
 LLVMRustWriteThinBitcodeToFile(LLVMPassManagerRef PMR,
                                LLVMModuleRef M,
-                               const char *BcFile)
+                               const char *BcFile
+                               size_t BcFileName)
 {
   report_fatal_error("ThinLTO not available");
 }
diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
index e045fb66..8e0b2b8b 100644
--- a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
+++ b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
@@ -9,14 +9,16 @@
 // except according to those terms.
 
 #include "rustllvm.h"
+#include "llvm/IR/CallSite.h"
+#include "llvm/IR/Constant.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/Type.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Bitcode/BitcodeWriterPass.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/Support/Casting.h"
 
 #if LLVM_VERSION_GE(5, 0)
@@ -36,6 +38,10 @@ using namespace llvm;
 using namespace llvm::sys;
 using namespace llvm::object;
 
+// This opcode is an LLVM detail that could hypothetically change (?), so
+// verify that the hard-coded value in `dwarf_const.rs` still agrees with LLVM.
+static_assert(dwarf::DW_OP_LLVM_fragment == 0x1000);
+
 // LLVMAtomicOrdering is already an enum - don't create another
 // one.
 static AtomicOrdering fromRust(LLVMAtomicOrdering Ordering)
@@ -71,11 +77,6 @@ extern "C" LLVMTypeRef LLVMRustGetFunctionReturnType(LLVMValueRef V)
   return wrap(dyn_cast<llvm::Function>(unwrap<llvm::Value>(V))->getReturnType());
 }
 
-extern "C" LLVMTypeRef LLVMRustGetValueType(LLVMValueRef V)
-{
-  return wrap(unwrap<llvm::Value>(V)->getType());
-}
-
 extern "C" LLVMValueRef
 LLVMRustInsertPrivateGlobal(LLVMModuleRef M, LLVMTypeRef Ty)
 {
@@ -94,6 +95,19 @@ extern "C" void LLVMRustSetFastMath(LLVMValueRef V)
   }
 }
 
+extern "C" void LLVMRustSetAlgebraicMath(LLVMValueRef V)
+{
+  if (auto I = dyn_cast<Instruction>(unwrap<Value>(V)))
+  {
+    I->setHasAllowReassoc(true);
+#if LLVM_VERSION_GE(11, 0)
+    I->setHasAllowContract(true);
+#endif
+    I->setHasAllowReciprocal(true);
+    I->setHasNoSignedZeros(true);
+  }
+}
+
 extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B,
                                             LLVMValueRef Dst, unsigned DstAlign,
                                             LLVMValueRef Src, unsigned SrcAlign,
@@ -172,10 +186,9 @@ extern "C" void LLVMRustPrintPassTimings()
   TimerGroup::printAll(OS);
 }
 
-extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M,
-                                              const char *Name)
+extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, size_t NameLen)
 {
-  return wrap(unwrap(M)->getNamedValue(Name));
+  return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen)));
 }
 
 extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M,
@@ -388,12 +401,12 @@ extern "C" void LLVMRustAddDereferenceableOrNullAttr(LLVMValueRef Fn,
 
 extern "C" void LLVMRustAddFunctionAttrStringValue(LLVMValueRef Fn,
                                                    unsigned Index,
-                                                   const char *Name,
-                                                   const char *Value)
+                                                   const char *Name, size_t NameLen,
+                                                   const char *Value, size_t ValueLen)
 {
   Function *F = unwrap<Function>(Fn);
   AttrBuilder B;
-  B.addAttribute(Name, Value);
+  B.addAttribute(StringRef(Name, NameLen), StringRef(Value, ValueLen));
 #if LLVM_VERSION_GE(5, 0)
   F->addAttributes(Index, B);
 #else
@@ -545,9 +558,9 @@ extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints,
                            StringRef(Constraints, ConstraintsLen));
 }
 
-extern "C" void LLVMRustAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm)
+extern "C" void LLVMRustAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm, size_t AsmLen)
 {
-  unwrap(M)->appendModuleInlineAsm(StringRef(Asm));
+  unwrap(M)->appendModuleInlineAsm(StringRef(Asm, AsmLen));
 }
 
 typedef DIBuilder *LLVMRustDIBuilderRef;
@@ -631,15 +644,9 @@ inline LLVMRustDIFlags visibility(LLVMRustDIFlags F)
   return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(F) & 0x3);
 }
 
-#if LLVM_VERSION_GE(4, 0)
 static DINode::DIFlags fromRust(LLVMRustDIFlags Flags)
 {
   DINode::DIFlags Result = DINode::DIFlags::FlagZero;
-#else
-static unsigned fromRust(LLVMRustDIFlags Flags)
-{
-  unsigned Result = 0;
-#endif
 
   switch (visibility(Flags))
   {
@@ -737,6 +744,44 @@ static unsigned fromRust(LLVMRustDIFlags Flags)
   return Result;
 }
 
+enum class LLVMRustDebugEmissionKind {
+  NoDebug,
+  FullDebug,
+  LineTablesOnly,
+};
+
+static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) {
+  switch (Kind) {
+  case LLVMRustDebugEmissionKind::NoDebug:
+    return DICompileUnit::DebugEmissionKind::NoDebug;
+  case LLVMRustDebugEmissionKind::FullDebug:
+    return DICompileUnit::DebugEmissionKind::FullDebug;
+  case LLVMRustDebugEmissionKind::LineTablesOnly:
+    return DICompileUnit::DebugEmissionKind::LineTablesOnly;
+  default:
+    report_fatal_error("bad DebugEmissionKind.");
+  }
+}
+
+enum class LLVMRustChecksumKind {
+  None,
+  MD5,
+  SHA1,
+};
+
+static Optional<DIFile::ChecksumKind> fromRust(LLVMRustChecksumKind Kind) {
+  switch (Kind) {
+  case LLVMRustChecksumKind::None:
+    return Optional<DIFile::ChecksumKind>{};
+  case LLVMRustChecksumKind::MD5:
+    return DIFile::ChecksumKind::CSK_MD5;
+  case LLVMRustChecksumKind::SHA1:
+    return DIFile::ChecksumKind::CSK_SHA1;
+  default:
+    report_fatal_error("bad ChecksumKind.");
+  }
+}
+
 extern "C" uint32_t LLVMRustDebugMetadataVersion()
 {
   return DEBUG_METADATA_VERSION;
@@ -774,26 +819,33 @@ extern "C" void LLVMRustDIBuilderFinalize(LLVMRustDIBuilderRef Builder)
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit(
     LLVMRustDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef,
-    const char *Producer, bool isOptimized, const char *Flags,
-    unsigned RuntimeVer, const char *SplitName)
+    const char *Producer, size_t ProducerLen, bool isOptimized, const char *Flags,
+    unsigned RuntimeVer, const char *SplitName, size_t SplitNameLen,
+    LLVMRustDebugEmissionKind Kind, uint64_t DWOId, bool SplitDebugInlining)
 {
   auto *File = unwrapDI<DIFile>(FileRef);
-
-#if LLVM_VERSION_GE(4, 0)
-  return wrap(Builder->createCompileUnit(Lang, File, Producer, isOptimized,
-                                         Flags, RuntimeVer, SplitName));
-#else
-  return wrap(Builder->createCompileUnit(Lang, File->getFilename(),
-                                         File->getDirectory(), Producer, isOptimized,
-                                         Flags, RuntimeVer, SplitName));
-#endif
+  return wrap(Builder->createCompileUnit(
+      Lang, File, StringRef(Producer, ProducerLen), isOptimized, Flags,
+      RuntimeVer, StringRef(SplitName, SplitNameLen), fromRust(Kind), DWOId,
+      SplitDebugInlining));
 }
 
-extern "C" LLVMMetadataRef
-LLVMRustDIBuilderCreateFile(LLVMRustDIBuilderRef Builder, const char *Filename,
-                            const char *Directory)
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile(
+  LLVMRustDIBuilderRef Builder, const char *Filename, size_t FilenameLen,
+  const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind,
+  const char *Checksum, size_t ChecksumLen, const char *Source, size_t SourceLen)
 {
-  return wrap(Builder->createFile(Filename, Directory));
+
+  Optional<DIFile::ChecksumKind> llvmCSKind = fromRust(CSKind);
+  Optional<DIFile::ChecksumInfo<StringRef>> CSInfo{};
+  if (llvmCSKind)
+    CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen});
+  Optional<StringRef> oSource{};
+  if (Source)
+    oSource = StringRef(Source, SourceLen);
+  return wrap(Builder->createFile(StringRef(Filename, FilenameLen),
+                                  StringRef(Directory, DirectoryLen), CSInfo,
+                                  oSource));
 }
 
 extern "C" LLVMMetadataRef
@@ -824,12 +876,24 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction(
 }
 
 extern "C" LLVMMetadataRef
-LLVMRustDIBuilderCreateBasicType(LLVMRustDIBuilderRef Builder, const char *Name,
+LLVMRustDIBuilderCreateBasicType(LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen,
                                  uint64_t SizeInBits, unsigned Encoding)
 {
-  return wrap(Builder->createBasicType(Name, SizeInBits, Encoding));
+  return wrap(Builder->createBasicType(StringRef(Name, NameLen), SizeInBits, Encoding));
 }
 
+extern "C" LLVMMetadataRef
+LLVMRustDIBuilderCreateTypedef(LLVMRustDIBuilderRef Builder,
+                               LLVMMetadataRef Type, const char *Name,
+                               size_t NameLen, LLVMMetadataRef File,
+                               unsigned LineNo, LLVMMetadataRef Scope)
+{
+  return wrap(Builder->createTypedef(
+      unwrap<DIType>(Type), StringRef(Name, NameLen), unwrap<DIFile>(File),
+      LineNo, unwrapDIPtr<DIScope>(Scope)));
+}
+
+
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType(
     LLVMRustDIBuilderRef Builder, LLVMMetadataRef PointeeTy,
     uint64_t SizeInBits, uint32_t AlignInBits, const char *Name)
@@ -843,41 +907,41 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType(
 }
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType(
-    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen,
     LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
     uint32_t AlignInBits, LLVMRustDIFlags Flags,
     LLVMMetadataRef DerivedFrom, LLVMMetadataRef Elements,
     unsigned RunTimeLang, LLVMMetadataRef VTableHolder,
-    const char *UniqueId)
+    const char *UniqueId, size_t UniqueIdLen)
 {
   return wrap(Builder->createStructType(
-      unwrapDI<DIDescriptor>(Scope), Name, unwrapDI<DIFile>(File), LineNumber,
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNumber,
       SizeInBits, AlignInBits, fromRust(Flags), unwrapDI<DIType>(DerivedFrom),
       DINodeArray(unwrapDI<MDTuple>(Elements)), RunTimeLang,
-      unwrapDI<DIType>(VTableHolder), UniqueId));
+      unwrapDI<DIType>(VTableHolder), StringRef(UniqueId, UniqueIdLen)));
 }
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart(
     LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
-    const char *Name,
+    const char *Name, size_t NameLen,
     LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
     uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Discriminator,
-    LLVMMetadataRef Elements, const char *UniqueId)
+    LLVMMetadataRef Elements, const char *UniqueId, size_t UniqueIdLen)
 {
   return wrap(Builder->createVariantPart(
-      unwrapDI<DIDescriptor>(Scope), Name,
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen),
       unwrapDI<DIFile>(File), LineNumber,
       SizeInBits, AlignInBits, fromRust(Flags), unwrapDI<DIDerivedType>(Discriminator),
-      DINodeArray(unwrapDI<MDTuple>(Elements)), UniqueId));
+      DINodeArray(unwrapDI<MDTuple>(Elements)), StringRef(UniqueId, UniqueIdLen)));
 }
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType(
     LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
-    LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits,
+    size_t NameLen, LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits,
     uint32_t AlignInBits, uint64_t OffsetInBits, LLVMRustDIFlags Flags,
     LLVMMetadataRef Ty)
 {
-  return wrap(Builder->createMemberType(unwrapDI<DIDescriptor>(Scope), Name,
+  return wrap(Builder->createMemberType(unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen),
                                         unwrapDI<DIFile>(File), LineNo,
                                         SizeInBits, AlignInBits, OffsetInBits,
                                         fromRust(Flags), unwrapDI<DIType>(Ty)));
@@ -885,7 +949,7 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType(
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType(
     LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
-    const char *Name, LLVMMetadataRef File, unsigned LineNo,
+    const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo,
     uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, LLVMValueRef Discriminant,
     LLVMRustDIFlags Flags, LLVMMetadataRef Ty)
 {
@@ -895,7 +959,7 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType(
     D = unwrap<llvm::ConstantInt>(Discriminant);
   }
   return wrap(Builder->createVariantMemberType(unwrapDI<DIDescriptor>(Scope),
-                                               Name,
+                                               StringRef(Name, NameLen),
                                                unwrapDI<DIFile>(File), LineNo,
                                                SizeInBits, AlignInBits, OffsetInBits, D,
                                                fromRust(Flags), unwrapDI<DIType>(Ty)));
@@ -920,13 +984,13 @@ LLVMRustDIBuilderCreateLexicalBlockFile(LLVMRustDIBuilderRef Builder,
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable(
     LLVMRustDIBuilderRef Builder, LLVMMetadataRef Context, const char *Name,
-    const char *LinkageName, LLVMMetadataRef File, unsigned LineNo,
-    LLVMMetadataRef Ty, bool IsLocalToUnit, LLVMValueRef V,
-    LLVMMetadataRef Decl = nullptr, uint32_t AlignInBits = 0)
+    size_t NameLen, const char *LinkageName, size_t LinkageNameLen,
+    LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty,
+    bool IsLocalToUnit, LLVMValueRef V, LLVMMetadataRef Decl = nullptr,
+    uint32_t AlignInBits = 0)
 {
   llvm::GlobalVariable *InitVal = cast<llvm::GlobalVariable>(unwrap(V));
 
-#if LLVM_VERSION_GE(4, 0)
   llvm::DIExpression *InitExpr = nullptr;
   if (llvm::ConstantInt *IntVal = llvm::dyn_cast<llvm::ConstantInt>(InitVal))
   {
@@ -941,19 +1005,14 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable(
   }
 
   llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression(
-      unwrapDI<DIDescriptor>(Context), Name, LinkageName,
+      unwrapDI<DIDescriptor>(Context), StringRef(Name, NameLen),
+      StringRef(LinkageName, LinkageNameLen),
       unwrapDI<DIFile>(File), LineNo, unwrapDI<DIType>(Ty), IsLocalToUnit,
       InitExpr, unwrapDIPtr<MDNode>(Decl), AlignInBits);
 
   InitVal->setMetadata("dbg", VarExpr);
 
   return wrap(VarExpr);
-#else
-  return wrap(Builder->createGlobalVariable(
-      unwrapDI<DIDescriptor>(Context), Name, LinkageName,
-      unwrapDI<DIFile>(File), LineNo, unwrapDI<DIType>(Ty), IsLocalToUnit,
-      InitVal, unwrapDIPtr<MDNode>(Decl)));
-#endif
 }
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable(
@@ -966,12 +1025,7 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable(
   { // DW_TAG_auto_variable
     return wrap(Builder->createAutoVariable(
         unwrapDI<DIDescriptor>(Scope), Name, unwrapDI<DIFile>(File), LineNo,
-        unwrapDI<DIType>(Ty), AlwaysPreserve, fromRust(Flags)
-#if LLVM_VERSION_GE(4, 0)
-                                                  ,
-        AlignInBits
-#endif
-        ));
+        unwrapDI<DIType>(Ty), AlwaysPreserve, fromRust(Flags), AlignInBits));
   }
   else
   {
@@ -1031,49 +1085,49 @@ extern "C" LLVMValueRef LLVMRustDIBuilderInsertDeclareAtEnd(
 
 extern "C" LLVMMetadataRef
 LLVMRustDIBuilderCreateEnumerator(LLVMRustDIBuilderRef Builder,
-                                  const char *Name, uint64_t Val)
+                                  const char *Name, size_t NameLen, int64_t Val,
+                                  bool IsUnsigned)
 {
-  return wrap(Builder->createEnumerator(Name, Val));
+  return wrap(Builder->createEnumerator(StringRef(Name, NameLen), Val, IsUnsigned));
 }
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType(
     LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
-    LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
-    uint32_t AlignInBits, LLVMMetadataRef Elements,
-    LLVMMetadataRef ClassTy)
+    size_t NameLen, LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
+    uint32_t AlignInBits, LLVMMetadataRef Elements, LLVMMetadataRef ClassTy)
 {
   return wrap(Builder->createEnumerationType(
-      unwrapDI<DIDescriptor>(Scope), Name, unwrapDI<DIFile>(File), LineNumber,
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNumber,
       SizeInBits, AlignInBits, DINodeArray(unwrapDI<MDTuple>(Elements)),
       unwrapDI<DIType>(ClassTy)));
 }
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType(
-    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, size_t NameLen,
     LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
     uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Elements,
-    unsigned RunTimeLang, const char *UniqueId)
+    unsigned RunTimeLang, const char *UniqueId, size_t UniqueIdLen)
 {
   return wrap(Builder->createUnionType(
-      unwrapDI<DIDescriptor>(Scope), Name, unwrapDI<DIFile>(File), LineNumber,
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), LineNumber,
       SizeInBits, AlignInBits, fromRust(Flags),
-      DINodeArray(unwrapDI<MDTuple>(Elements)), RunTimeLang, UniqueId));
+      DINodeArray(unwrapDI<MDTuple>(Elements)), RunTimeLang, StringRef(UniqueId, UniqueIdLen)));
 }
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTemplateTypeParameter(
     LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
-    LLVMMetadataRef Ty)
+    size_t NameLen, LLVMMetadataRef Ty)
 {
   return wrap(Builder->createTemplateTypeParameter(
-      unwrapDI<DIDescriptor>(Scope), Name, unwrapDI<DIType>(Ty)));
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIType>(Ty)));
 }
 
 extern "C" LLVMMetadataRef
 LLVMRustDIBuilderCreateNameSpace(LLVMRustDIBuilderRef Builder,
-                                 LLVMMetadataRef Scope, const char *Name)
+                                 LLVMMetadataRef Scope, const char *Name, size_t NameLen)
 {
   return wrap(Builder->createNameSpace(
-      unwrapDI<DIDescriptor>(Scope), Name
+      unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen)
 #if LLVM_VERSION_GE(4, 0)
       ,
       false // ExportSymbols (only relevant for C++ anonymous namespaces)
@@ -1112,18 +1166,15 @@ LLVMRustDIBuilderCreateDebugLocation(unsigned Line,
   return wrap(debug_loc.getAsMDNode());
 }
 
-extern "C" int64_t LLVMRustDIBuilderCreateOpDeref()
-{
-  return dwarf::DW_OP_deref;
-}
-
-extern "C" int64_t LLVMRustDIBuilderCreateOpPlusUconst()
-{
-#if LLVM_VERSION_GE(5, 0)
-  return dwarf::DW_OP_plus_uconst;
+extern "C" LLVMMetadataRef
+LLVMRustDILocationCloneWithBaseDiscriminator(LLVMMetadataRef Location,
+                                             unsigned BD) {
+  DILocation *Loc = unwrapDIPtr<DILocation>(Location);
+#if LLVM_VERSION_LT(8, 0)
+  return wrap(Loc->setBaseDiscriminator(BD));
 #else
-  // older LLVM used `plus` to behave like `plus_uconst`.
-  return dwarf::DW_OP_plus;
+  auto NewLoc = Loc->cloneWithBaseDiscriminator(BD);
+  return wrap(NewLoc.has_value() ? NewLoc.value() : nullptr);
 #endif
 }
 
@@ -1842,4 +1893,36 @@ extern "C" LLVMValueRef
 {
   return nullptr;
 }
+#endif
+
+// Backport for LLVM < 8.0
+#if LLVM_VERSION_LT(8, 0)
+extern "C" LLVMValueRef LLVMConstInBoundsGEP2(LLVMTypeRef Ty,
+                                              LLVMValueRef ConstantVal,
+                                              LLVMValueRef *ConstantIndices,
+                                              unsigned NumIndices)
+{
+  ArrayRef<Constant *> IdxList(unwrap<Constant>(ConstantIndices, NumIndices),
+                               NumIndices);
+  Constant* Val = unwrap<Constant>(ConstantVal);
+  return wrap(ConstantExpr::getInBoundsGetElementPtr(unwrap(Ty), Val, IdxList));
+}
+
+extern "C" LLVMValueRef LLVMBuildGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                      LLVMValueRef Pointer, LLVMValueRef *Indices,
+                                      unsigned NumIndices, const char *Name)
+{
+  ArrayRef<Value *> IdxList(unwrap(Indices), NumIndices);
+  return wrap(unwrap(B)->CreateGEP(unwrap(Ty), unwrap(Pointer), IdxList, Name));
+}
+
+extern "C" LLVMValueRef LLVMBuildInBoundsGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                              LLVMValueRef Pointer, LLVMValueRef *Indices,
+                                              unsigned NumIndices, const char *Name)
+{
+  ArrayRef<Value *> IdxList(unwrap(Indices), NumIndices);
+  return wrap(
+    unwrap(B)->CreateInBoundsGEP(unwrap(Ty), unwrap(Pointer), IdxList, Name));
+}
+
 #endif
\ No newline at end of file
diff --git a/crates/rustc_codegen_nvvm/src/abi.rs b/crates/rustc_codegen_nvvm/src/abi.rs
index f9adb1bc..58578990 100644
--- a/crates/rustc_codegen_nvvm/src/abi.rs
+++ b/crates/rustc_codegen_nvvm/src/abi.rs
@@ -1,23 +1,28 @@
-use crate::builder::Builder;
-use crate::context::CodegenCx;
-use crate::int_replace::{get_transformed_type, transmute_llval};
-use crate::llvm::{self, *};
-use crate::ty::LayoutLlvmExt;
+use std::cmp;
+
 use libc::c_uint;
+use rustc_abi::BackendRepr::Scalar;
+use rustc_abi::Size;
+use rustc_abi::{HasDataLayout, Primitive, Reg, RegKind};
+use rustc_codegen_ssa::mir::operand::OperandRef;
 use rustc_codegen_ssa::mir::operand::OperandValue;
-use rustc_codegen_ssa::mir::place::PlaceRef;
-use rustc_codegen_ssa::traits::BaseTypeMethods;
-use rustc_codegen_ssa::{traits::*, MemFlags};
+use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
+use rustc_codegen_ssa::{MemFlags, traits::*};
 use rustc_middle::bug;
 use rustc_middle::ty::layout::LayoutOf;
-pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
+pub use rustc_middle::ty::layout::{WIDE_PTR_ADDR, WIDE_PTR_EXTRA};
 use rustc_middle::ty::{Ty, TyCtxt, TyKind};
-pub use rustc_target::abi::call::*;
-use rustc_target::abi::call::{CastTarget, Reg, RegKind};
-use rustc_target::abi::{self, HasDataLayout, Int};
-pub use rustc_target::spec::abi::Abi;
+use rustc_target::callconv::{
+    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, Conv, FnAbi, PassMode,
+};
 use tracing::trace;
 
+use crate::builder::Builder;
+use crate::context::CodegenCx;
+use crate::int_replace::{get_transformed_type, transmute_llval};
+use crate::llvm::{self, *};
+use crate::ty::LayoutLlvmExt;
+
 pub(crate) fn readjust_fn_abi<'tcx>(
     tcx: TyCtxt<'tcx>,
     fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
@@ -29,8 +34,7 @@ pub(crate) fn readjust_fn_abi<'tcx>(
     let readjust_arg_abi = |arg: &ArgAbi<'tcx, Ty<'tcx>>| {
         let mut arg = ArgAbi {
             layout: arg.layout,
-            mode: arg.mode,
-            pad: arg.pad,
+            mode: arg.mode.clone(),
         };
 
         // ignore zsts
@@ -193,47 +197,53 @@ impl LlvmType for Reg {
 impl LlvmType for CastTarget {
     fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type {
         let rest_ll_unit = self.rest.unit.llvm_type(cx);
-        let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 {
-            (0, 0)
+        let rest_count = if self.rest.total == Size::ZERO {
+            0
         } else {
-            (
-                self.rest.total.bytes() / self.rest.unit.size.bytes(),
-                self.rest.total.bytes() % self.rest.unit.size.bytes(),
-            )
+            assert_ne!(
+                self.rest.unit.size,
+                Size::ZERO,
+                "total size {:?} cannot be divided into units of zero size",
+                self.rest.total
+            );
+            if self.rest.total.bytes() % self.rest.unit.size.bytes() != 0 {
+                assert_eq!(
+                    self.rest.unit.kind,
+                    RegKind::Integer,
+                    "only int regs can be split"
+                );
+            }
+            self.rest
+                .total
+                .bytes()
+                .div_ceil(self.rest.unit.size.bytes())
         };
 
+        // Simplify to a single unit or an array if there's no prefix.
+        // This produces the same layout, but using a simpler type.
         if self.prefix.iter().all(|x| x.is_none()) {
-            // Simplify to a single unit when there is no prefix and size <= unit size
-            if self.rest.total <= self.rest.unit.size {
+            // We can't do this if is_consecutive is set and the unit would get
+            // split on the target. Currently, this is only relevant for i128
+            // registers.
+            if rest_count == 1 && (!self.rest.is_consecutive || self.rest.unit != Reg::i128()) {
                 return rest_ll_unit;
             }
 
-            // Simplify to array when all chunks are the same size and type
-            if rem_bytes == 0 {
-                return cx.type_array(rest_ll_unit, rest_count);
-            }
+            return cx.type_array(rest_ll_unit, rest_count);
         }
 
-        // Create list of fields in the main structure
-        let mut args: Vec<_> = self
+        // Generate a struct type with the prefix and the "rest" arguments.
+        let prefix_args = self
             .prefix
             .iter()
-            .flat_map(|option_reg| option_reg.map(|reg| reg.llvm_type(cx)))
-            .chain((0..rest_count).map(|_| rest_ll_unit))
-            .collect();
-
-        // Append final integer
-        if rem_bytes != 0 {
-            // Only integers can be really split further.
-            assert_eq!(self.rest.unit.kind, RegKind::Integer);
-            args.push(cx.type_ix(rem_bytes * 8));
-        }
-
+            .flat_map(|option_reg| option_reg.map(|reg| reg.llvm_type(cx)));
+        let rest_args = (0..rest_count).map(|_| rest_ll_unit);
+        let args: Vec<_> = prefix_args.chain(rest_args).collect();
         cx.type_struct(&args, false)
     }
 }
 
-impl<'a, 'll, 'tcx> ArgAbiMethods<'tcx> for Builder<'a, 'll, 'tcx> {
+impl<'a, 'll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
     fn store_fn_arg(
         &mut self,
         arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
@@ -264,27 +274,32 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
 
 impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
     fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
-        let args_capacity: usize = self.args.iter().map(|arg|
-            if arg.pad.is_some() { 1 } else { 0 } +
-            if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 }
-        ).sum();
+        // Ignore "extra" args when calling C variadic functions.
+        // Only the "fixed" args are part of the LLVM function signature.
+        let args = if self.c_variadic {
+            &self.args[..self.fixed_count as usize]
+        } else {
+            &self.args
+        };
+
+        // This capacity calculation is approximate.
+        let mut llargument_tys = Vec::with_capacity(
+            self.args.len()
+                + if let PassMode::Indirect { .. } = self.ret.mode {
+                    1
+                } else {
+                    0
+                },
+        );
 
         // the current index of each parameter in the function. Cant use enumerate on args because
         // some pass modes pass args as multiple params, such as scalar pairs.
         let mut idx = 0;
 
-        let mut llargument_tys = Vec::with_capacity(
-            if let PassMode::Indirect { .. } = self.ret.mode {
-                1
-            } else {
-                0
-            } + args_capacity,
-        );
-
-        let mut llreturn_ty = match self.ret.mode {
+        let mut llreturn_ty = match &self.ret.mode {
             PassMode::Ignore => cx.type_void(),
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
-            PassMode::Cast(cast) => cast.llvm_type(cx),
+            PassMode::Cast { cast, .. } => cast.llvm_type(cx),
             PassMode::Indirect { .. } => {
                 idx += 1;
                 llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx)));
@@ -301,14 +316,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             old_ret_ty = None;
         }
 
-        for arg in self.args.iter() {
-            // add padding
-            if let Some(ty) = arg.pad {
-                idx += 1;
-                llargument_tys.push(ty.llvm_type(cx));
-            }
-
-            let llarg_ty = match arg.mode {
+        for arg in args {
+            let llarg_ty = match &arg.mode {
                 PassMode::Ignore => continue,
                 PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx),
                 PassMode::Pair(..) => {
@@ -319,20 +328,27 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 }
                 PassMode::Indirect {
                     attrs: _,
-                    extra_attrs: Some(_),
+                    meta_attrs: Some(_),
                     on_stack: _,
                 } => {
-                    let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty);
+                    let ptr_ty = Ty::new_mut_ptr(cx.tcx, arg.layout.ty);
                     let ptr_layout = cx.layout_of(ptr_ty);
                     llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true));
                     llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true));
                     idx += 2;
                     continue;
                 }
-                PassMode::Cast(cast) => cast.llvm_type(cx),
+                PassMode::Cast { cast, pad_i32 } => {
+                    // add padding
+                    if *pad_i32 {
+                        idx += 1;
+                        llargument_tys.push(Reg::i32().llvm_type(cx));
+                    }
+                    cast.llvm_type(cx)
+                }
                 PassMode::Indirect {
                     attrs: _,
-                    extra_attrs: None,
+                    meta_attrs: None,
                     on_stack: _,
                 } => cx.type_ptr_to(arg.memory_ty(cx)),
             };
@@ -350,6 +366,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             cx.type_func(&llargument_tys, llreturn_ty)
         };
         if !transformed_types.is_empty() || old_ret_ty.is_some() {
+            trace!("remapping args in {:?} to {:?}", ty, transformed_types);
             cx.remapped_integer_args
                 .borrow_mut()
                 .insert(ty, (old_ret_ty, transformed_types));
@@ -367,7 +384,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
     }
 
     fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) {
-        if self.ret.layout.abi.is_uninhabited() {
+        if self.ret.layout.is_uninhabited() {
             llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn);
         }
 
@@ -383,13 +400,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             i += 1;
             i - 1
         };
-        match self.ret.mode {
-            PassMode::Direct(ref attrs) => {
+        match &self.ret.mode {
+            PassMode::Direct(attrs) => {
                 attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
             }
             PassMode::Indirect {
-                ref attrs,
-                extra_attrs: _,
+                attrs,
+                meta_attrs: _,
                 on_stack,
             } => {
                 assert!(!on_stack);
@@ -399,14 +416,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             _ => {}
         }
         for arg in &self.args {
-            if arg.pad.is_some() {
-                apply(&ArgAttributes::new());
-            }
-            match arg.mode {
+            match &arg.mode {
                 PassMode::Ignore => {}
                 PassMode::Indirect {
-                    ref attrs,
-                    extra_attrs: None,
+                    attrs,
+                    meta_attrs: None,
                     on_stack: true,
                 } => {
                     apply(attrs);
@@ -415,29 +429,32 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     // C++ API, so somebody more experienced in the C++ API should look at this.
                     // it shouldnt do anything bad since it seems to be only for optimization.
                 }
-                PassMode::Direct(ref attrs)
+                PassMode::Direct(attrs)
                 | PassMode::Indirect {
-                    ref attrs,
-                    extra_attrs: None,
+                    attrs,
+                    meta_attrs: None,
                     on_stack: false,
                 } => {
                     apply(attrs);
                 }
                 PassMode::Indirect {
-                    ref attrs,
-                    extra_attrs: Some(ref extra_attrs),
+                    attrs,
+                    meta_attrs: Some(extra_attrs),
                     on_stack,
                 } => {
                     assert!(!on_stack);
                     apply(attrs);
                     apply(extra_attrs);
                 }
-                PassMode::Pair(ref a, ref b) => {
+                PassMode::Pair(a, b) => {
                     apply(a);
                     apply(b);
                 }
-                PassMode::Cast(_) => {
-                    apply(&ArgAttributes::new());
+                PassMode::Cast { cast, pad_i32 } => {
+                    if *pad_i32 {
+                        apply(&ArgAttributes::new());
+                    }
+                    apply(&cast.attrs);
                 }
             }
         }
@@ -459,63 +476,64 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             i - 1
         };
         match self.ret.mode {
-            PassMode::Direct(ref attrs) => {
+            PassMode::Direct(attrs) => {
                 attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite);
             }
             PassMode::Indirect {
-                ref attrs,
-                extra_attrs: _,
+                attrs,
+                meta_attrs: _,
                 on_stack,
             } => {
                 assert!(!on_stack);
-                apply(bx.cx, attrs);
+                apply(bx.cx, &attrs);
             }
             _ => {}
         }
-        if let abi::Abi::Scalar(ref scalar) = self.ret.layout.abi {
+        if let Scalar(scalar) = self.ret.layout.backend_repr {
             // If the value is a boolean, the range is 0..2 and that ultimately
             // become 0..0 when the type becomes i1, which would be rejected
             // by the LLVM verifier.
-            if let Int(..) = scalar.value {
+            if let Primitive::Int(..) = scalar.primitive() {
                 if !scalar.is_bool() && !scalar.is_always_valid(bx) {
-                    bx.range_metadata(callsite, scalar.valid_range);
+                    trace!("apply_attrs_callsite -> range_metadata");
+                    bx.range_metadata(callsite, scalar.valid_range(bx));
                 }
             }
         }
-        for arg in &self.args {
-            if arg.pad.is_some() {
-                apply(bx.cx, &ArgAttributes::new());
-            }
-            match arg.mode {
+        for arg in self.args.iter() {
+            match &arg.mode {
                 PassMode::Ignore => {}
                 PassMode::Indirect {
-                    ref attrs,
-                    extra_attrs: None,
+                    attrs,
+                    meta_attrs: None,
                     on_stack: true,
                 } => {
                     apply(bx.cx, attrs);
                 }
-                PassMode::Direct(ref attrs)
+                PassMode::Direct(attrs)
                 | PassMode::Indirect {
-                    ref attrs,
-                    extra_attrs: None,
+                    attrs,
+                    meta_attrs: None,
                     on_stack: false,
                 } => {
                     apply(bx.cx, attrs);
                 }
                 PassMode::Indirect {
-                    ref attrs,
-                    extra_attrs: Some(ref extra_attrs),
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
                     on_stack: _,
                 } => {
                     apply(bx.cx, attrs);
-                    apply(bx.cx, extra_attrs);
+                    apply(bx.cx, meta_attrs);
                 }
-                PassMode::Pair(ref a, ref b) => {
+                PassMode::Pair(a, b) => {
                     apply(bx.cx, a);
                     apply(bx.cx, b);
                 }
-                PassMode::Cast(_) => {
+                PassMode::Cast { pad_i32, .. } => {
+                    if *pad_i32 {
+                        apply(bx.cx, &ArgAttributes::new());
+                    }
                     apply(bx.cx, &ArgAttributes::new());
                 }
             }
@@ -524,15 +542,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
 }
 
 impl<'a, 'll, 'tcx> AbiBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
-    fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value) {
-        fn_abi.apply_attrs_callsite(self, callsite)
-    }
-
     fn get_param(&mut self, index: usize) -> Self::Value {
         let val = llvm::get_param(self.llfn(), index as c_uint);
-        trace!("Get param `{:?}`", val);
-        unsafe {
+        // trace!("Get param `{:?}`", val);
+        let val = unsafe {
             let llfnty = LLVMRustGetFunctionType(self.llfn());
+            trace!("llfnty: {:?}", llfnty);
             // destructure so rustc doesnt complain in the call to transmute_llval
             let Self { cx, llbuilder } = self;
             let map = cx.remapped_integer_args.borrow();
@@ -543,7 +558,9 @@ impl<'a, 'll, 'tcx> AbiBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                 }
             }
             val
-        }
+        };
+        trace!("Get param `{:?}`", val);
+        val
     }
 }
 
@@ -580,41 +597,63 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
         val: &'ll Value,
         dst: PlaceRef<'tcx, &'ll Value>,
     ) {
-        if self.is_ignore() {
-            return;
-        }
-
-        if self.is_sized_indirect() {
-            OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst)
-        } else if self.is_unsized_indirect() {
-            bug!("unsized `ArgAbi` must be handled through `store_fn_arg`");
-        } else if let PassMode::Cast(cast) = self.mode {
-            let can_store_through_cast_ptr = false;
-            if can_store_through_cast_ptr {
-                let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx));
-                let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty);
-                bx.store(val, cast_dst, self.layout.align.abi);
-            } else {
+        match &self.mode {
+            PassMode::Ignore => {}
+            // Sized indirect arguments
+            PassMode::Indirect {
+                attrs,
+                meta_attrs: None,
+                on_stack: _,
+            } => {
+                let align = attrs.pointee_align.unwrap_or(self.layout.align.abi);
+                OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst);
+            }
+            // Unsized indirect arguments
+            PassMode::Indirect {
+                attrs: _,
+                meta_attrs: Some(_),
+                on_stack: _,
+            } => {
+                bug!("unsized `ArgAbi` must be handled through `store_fn_arg`");
+            }
+            PassMode::Cast { pad_i32: _, cast } => {
+                trace!("store cast");
+                // The ABI mandates that the value is passed as a different struct representation.
+                // Spill and reload it from the stack to convert from the ABI representation to
+                // the Rust representation.
                 let scratch_size = cast.size(bx);
                 let scratch_align = cast.align(bx);
-                let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align);
+                // Note that the ABI type may be either larger or smaller than the Rust type,
+                // due to the presence or absence of trailing padding. For example:
+                // - On some ABIs, the Rust layout { f64, f32, <f32 padding> } may omit padding
+                //   when passed by value, making it smaller.
+                // - On some ABIs, the Rust layout { u16, u16, u16 } may be padded up to 8 bytes
+                //   when passed by value, making it larger.
+                let copy_bytes =
+                    cmp::min(cast.unaligned_size(bx).bytes(), self.layout.size.bytes());
+                // Allocate some scratch space...
+                let llscratch = bx.alloca(scratch_size, scratch_align);
                 bx.lifetime_start(llscratch, scratch_size);
-
+                // ...store the value...
                 bx.store(val, llscratch, scratch_align);
-
+                // ... and then memcpy it to the intended destination.
                 bx.memcpy(
-                    dst.llval,
+                    dst.val.llval,
                     self.layout.align.abi,
                     llscratch,
                     scratch_align,
-                    bx.const_usize(self.layout.size.bytes()),
+                    bx.const_usize(copy_bytes),
                     MemFlags::empty(),
                 );
 
                 bx.lifetime_end(llscratch, scratch_size);
+                trace!("store cast end");
+            }
+            _ => {
+                OperandRef::from_immediate_or_packed_pair(bx, val, self.layout)
+                    .val
+                    .store(bx, dst);
             }
-        } else {
-            OperandValue::Immediate(val).store(bx, dst);
         }
     }
 
@@ -636,18 +675,23 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             }
             PassMode::Indirect {
                 attrs: _,
-                extra_attrs: Some(_),
+                meta_attrs: Some(_),
                 on_stack: _,
             } => {
-                OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst);
+                let place_val = PlaceValue {
+                    llval: next(),
+                    llextra: Some(next()),
+                    align: self.layout.align.abi,
+                };
+                OperandValue::Ref(place_val).store(bx, dst);
             }
             PassMode::Direct(_)
             | PassMode::Indirect {
                 attrs: _,
-                extra_attrs: None,
+                meta_attrs: None,
                 on_stack: _,
             }
-            | PassMode::Cast(_) => {
+            | PassMode::Cast { .. } => {
                 let next_arg = next();
                 self.store(bx, next_arg, dst);
             }
diff --git a/crates/rustc_codegen_nvvm/src/allocator.rs b/crates/rustc_codegen_nvvm/src/allocator.rs
index c0090d76..edb9bea9 100644
--- a/crates/rustc_codegen_nvvm/src/allocator.rs
+++ b/crates/rustc_codegen_nvvm/src/allocator.rs
@@ -1,141 +1,162 @@
+use crate::LlvmMod;
 use crate::llvm::{self, False, True};
 use crate::target;
-use crate::LlvmMod;
 use libc::c_uint;
-use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
+use rustc_ast::expand::allocator::{
+    ALLOCATOR_METHODS, AllocatorKind, AllocatorTy, alloc_error_handler_name, default_fn_name,
+};
 use rustc_middle::ty::TyCtxt;
-use rustc_span::symbol::sym;
 
 // adapted from rustc_codegen_llvm
 pub(crate) unsafe fn codegen(
     _tcx: TyCtxt<'_>,
     mods: &mut LlvmMod,
+    _module_name: &str,
     kind: AllocatorKind,
-    has_alloc_error_handler: bool,
+    alloc_error_handler_kind: AllocatorKind,
 ) {
     let llcx = &*mods.llcx;
-    let llmod = mods.llmod.as_ref().unwrap();
-    let usize = target::usize_ty(llcx);
-    let i8 = llvm::LLVMInt8TypeInContext(llcx);
-    let i8p = llvm::LLVMPointerType(i8, 0);
-    let void = llvm::LLVMVoidTypeInContext(llcx);
+    let llmod = unsafe { mods.llmod.as_ref().unwrap() };
+    let usize = unsafe { target::usize_ty(llcx) };
+    let i8 = unsafe { llvm::LLVMInt8TypeInContext(llcx) };
+    let i8p = unsafe { llvm::LLVMPointerType(i8, 0) };
+    let void = unsafe { llvm::LLVMVoidTypeInContext(llcx) };
 
     let mut used = Vec::new();
 
-    for method in ALLOCATOR_METHODS {
-        let mut args = Vec::with_capacity(method.inputs.len());
-        for ty in method.inputs.iter() {
-            match *ty {
-                AllocatorTy::Layout => {
-                    args.push(usize); // size
-                    args.push(usize); // align
+    if kind == AllocatorKind::Default {
+        for method in ALLOCATOR_METHODS {
+            let mut args = Vec::with_capacity(method.inputs.len());
+            for ty in method.inputs.iter() {
+                match ty.ty {
+                    AllocatorTy::Layout => {
+                        args.push(usize); // size
+                        args.push(usize); // align
+                    }
+                    AllocatorTy::Ptr => args.push(i8p),
+                    AllocatorTy::Usize => args.push(usize),
+
+                    AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"),
                 }
-                AllocatorTy::Ptr => args.push(i8p),
-                AllocatorTy::Usize => args.push(usize),
-
-                AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"),
             }
-        }
-        let output = match method.output {
-            AllocatorTy::ResultPtr => Some(i8p),
-            AllocatorTy::Unit => None,
+            let output = match method.output {
+                AllocatorTy::ResultPtr => Some(i8p),
+                AllocatorTy::Unit => None,
 
-            AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => {
-                panic!("invalid allocator output")
+                AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => {
+                    panic!("invalid allocator output")
+                }
+            };
+
+            let ty = unsafe {
+                llvm::LLVMFunctionType(
+                    output.unwrap_or(void),
+                    args.as_ptr(),
+                    args.len() as c_uint,
+                    False,
+                )
+            };
+            let name = format!("__rust_{}", method.name);
+            let llfn = unsafe {
+                llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty)
+            };
+
+            used.push(llfn);
+            // nvvm doesnt support uwtable so dont try to generate it
+
+            let callee = default_fn_name(method.name);
+            let callee = unsafe {
+                llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty)
+            };
+            unsafe { llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden) };
+
+            let llbb = unsafe {
+                llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast())
+            };
+
+            let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(llcx) };
+            unsafe { llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb) };
+            let args = args
+                .iter()
+                .enumerate()
+                .map(|(i, _)| unsafe { llvm::LLVMGetParam(llfn, i as c_uint) })
+                .collect::<Vec<_>>();
+            let ret = unsafe {
+                llvm::LLVMRustBuildCall(
+                    llbuilder,
+                    callee,
+                    args.as_ptr(),
+                    args.len() as c_uint,
+                    None,
+                )
+            };
+            unsafe { llvm::LLVMSetTailCall(ret, True) };
+            if output.is_some() {
+                unsafe { llvm::LLVMBuildRet(llbuilder, ret) };
+            } else {
+                unsafe { llvm::LLVMBuildRetVoid(llbuilder) };
             }
-        };
-
-        let ty = llvm::LLVMFunctionType(
-            output.unwrap_or(void),
-            args.as_ptr(),
-            args.len() as c_uint,
-            False,
-        );
-        let name = format!("__rust_{}", method.name);
-        let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty);
-
-        used.push(llfn);
-        // nvvm doesnt support uwtable so dont try to generate it
-
-        let callee = kind.fn_name(method.name);
-        let callee =
-            llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty);
-        llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden);
-
-        let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast());
-
-        let llbuilder = llvm::LLVMCreateBuilderInContext(llcx);
-        llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb);
-        let args = args
-            .iter()
-            .enumerate()
-            .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint))
-            .collect::<Vec<_>>();
-        let ret =
-            llvm::LLVMRustBuildCall(llbuilder, callee, args.as_ptr(), args.len() as c_uint, None);
-        llvm::LLVMSetTailCall(ret, True);
-        if output.is_some() {
-            llvm::LLVMBuildRet(llbuilder, ret);
-        } else {
-            llvm::LLVMBuildRetVoid(llbuilder);
+            unsafe { llvm::LLVMDisposeBuilder(llbuilder) };
         }
-        llvm::LLVMDisposeBuilder(llbuilder);
     }
 
     // rust alloc error handler
     let args = [usize, usize]; // size, align
 
-    let ty = llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False);
+    let ty = unsafe { llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False) };
     let name = "__rust_alloc_error_handler".to_string();
-    let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty);
+    let llfn =
+        unsafe { llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty) };
 
     used.push(llfn);
 
     // -> ! DIFlagNoReturn
     llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn);
 
-    let kind = if has_alloc_error_handler {
-        AllocatorKind::Global
-    } else {
-        AllocatorKind::Default
+    let callee = alloc_error_handler_name(alloc_error_handler_kind);
+    let callee = unsafe {
+        llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty)
     };
-    let callee = kind.fn_name(sym::oom);
-    let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty);
 
     used.push(callee);
 
     // -> ! DIFlagNoReturn
     llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, callee);
-    llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden);
+    unsafe { llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden) };
 
-    let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast());
+    let llbb =
+        unsafe { llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()) };
 
-    let llbuilder = llvm::LLVMCreateBuilderInContext(llcx);
-    llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb);
+    let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(llcx) };
+    unsafe { llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb) };
     let args = args
         .iter()
         .enumerate()
-        .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint))
+        .map(|(i, _)| unsafe { llvm::LLVMGetParam(llfn, i as c_uint) })
         .collect::<Vec<_>>();
-    let ret = llvm::LLVMRustBuildCall(llbuilder, callee, args.as_ptr(), args.len() as c_uint, None);
-    llvm::LLVMSetTailCall(ret, True);
-    llvm::LLVMBuildRetVoid(llbuilder);
-    llvm::LLVMDisposeBuilder(llbuilder);
+    let ret = unsafe {
+        llvm::LLVMRustBuildCall(llbuilder, callee, args.as_ptr(), args.len() as c_uint, None)
+    };
+    unsafe { llvm::LLVMSetTailCall(ret, True) };
+    unsafe { llvm::LLVMBuildRetVoid(llbuilder) };
+    unsafe { llvm::LLVMDisposeBuilder(llbuilder) };
 
-    let ptr_ty = llvm::LLVMPointerType(llvm::LLVMInt8TypeInContext(llcx), 0);
+    let ptr_ty = unsafe { llvm::LLVMPointerType(llvm::LLVMInt8TypeInContext(llcx), 0) };
 
     for used in &mut used {
-        *used = llvm::LLVMConstBitCast(*used, ptr_ty);
+        *used = unsafe { llvm::LLVMConstBitCast(*used, ptr_ty) };
     }
 
-    let section = "llvm.metadata\0".as_ptr().cast();
-    let array = llvm::LLVMConstArray(ptr_ty, used.as_ptr(), used.len() as u32);
-    let g = llvm::LLVMAddGlobal(
-        llmod,
-        llvm::LLVMTypeOf(array),
-        "llvm.used\0".as_ptr().cast(),
-    );
-    llvm::LLVMSetInitializer(g, array);
-    llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage);
-    llvm::LLVMSetSection(g, section);
+    let section = c"llvm.metadata";
+    let array = unsafe { llvm::LLVMConstArray(ptr_ty, used.as_ptr(), used.len() as u32) };
+    let g = unsafe {
+        llvm::LLVMAddGlobal(
+            llmod,
+            llvm::LLVMTypeOf(array),
+            "llvm.used\0".as_ptr().cast(),
+        )
+    };
+    unsafe { llvm::LLVMSetInitializer(g, array) };
+    unsafe { llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage) };
+    unsafe { llvm::LLVMSetSection(g, section.as_ptr()) };
 }
diff --git a/crates/rustc_codegen_nvvm/src/asm.rs b/crates/rustc_codegen_nvvm/src/asm.rs
index 1a1c79f8..8d2e57ae 100644
--- a/crates/rustc_codegen_nvvm/src/asm.rs
+++ b/crates/rustc_codegen_nvvm/src/asm.rs
@@ -1,19 +1,19 @@
 use std::os::raw::{c_char, c_uint};
 
+use crate::common::AsCCharPtr;
 use crate::{
     llvm::{self, Value},
     ty::LayoutLlvmExt,
 };
-use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece, LlvmAsmDialect};
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_codegen_ssa::{
-    mir::{operand::OperandValue, place::PlaceRef},
+    mir::operand::OperandValue,
     traits::{
-        AsmBuilderMethods, AsmMethods, BaseTypeMethods, BuilderMethods, ConstMethods,
-        GlobalAsmOperandRef, InlineAsmOperandRef,
+        AsmBuilderMethods, AsmCodegenMethods, BaseTypeCodegenMethods, BuilderMethods,
+        ConstCodegenMethods, GlobalAsmOperandRef, InlineAsmOperandRef,
     },
 };
 use rustc_hash::FxHashMap;
-use rustc_hir::LlvmInlineAsmInner;
 use rustc_middle::{span_bug, ty::Instance};
 use rustc_span::{Pos, Span};
 use rustc_target::asm::{InlineAsmRegClass, InlineAsmRegOrRegClass, NvptxInlineAsmRegClass};
@@ -21,97 +21,6 @@ use rustc_target::asm::{InlineAsmRegClass, InlineAsmRegOrRegClass, NvptxInlineAs
 use crate::{builder::Builder, context::CodegenCx};
 
 impl<'a, 'll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
-    // inline llvm ir is allowed, but the user needs to make sure it doesnt violate
-    // nvvm ir constraints
-    fn codegen_llvm_inline_asm(
-        &mut self,
-        ia: &LlvmInlineAsmInner,
-        outputs: Vec<PlaceRef<'tcx, &'ll Value>>,
-        mut inputs: Vec<&'ll Value>,
-        span: Span,
-    ) -> bool {
-        let mut ext_constraints = vec![];
-        let mut output_types = vec![];
-
-        // Prepare the output operands
-        let mut indirect_outputs = vec![];
-        for (i, (out, &place)) in ia.outputs.iter().zip(&outputs).enumerate() {
-            if out.is_rw {
-                let operand = self.load_operand(place);
-                if let OperandValue::Immediate(_) = operand.val {
-                    inputs.push(operand.immediate());
-                }
-                ext_constraints.push(i.to_string());
-            }
-            if out.is_indirect {
-                let operand = self.load_operand(place);
-                if let OperandValue::Immediate(_) = operand.val {
-                    indirect_outputs.push(operand.immediate());
-                }
-            } else {
-                output_types.push(place.layout.llvm_type(self.cx));
-            }
-        }
-        if !indirect_outputs.is_empty() {
-            indirect_outputs.extend_from_slice(&inputs);
-            inputs = indirect_outputs;
-        }
-
-        let clobbers = ia.clobbers.iter().map(|s| format!("~{{{}}}", &s));
-
-        let all_constraints = ia
-            .outputs
-            .iter()
-            .map(|out| out.constraint.to_string())
-            .chain(ia.inputs.iter().map(|s| s.to_string()))
-            .chain(ext_constraints)
-            .chain(clobbers)
-            .collect::<Vec<String>>()
-            .join(",");
-
-        // Depending on how many outputs we have, the return type is different
-        let num_outputs = output_types.len();
-        let output_type = match num_outputs {
-            0 => self.type_void(),
-            1 => output_types[0],
-            _ => self.type_struct(&output_types, false),
-        };
-
-        let asm = ia.asm.as_str();
-        let r = inline_asm_call(
-            self,
-            &asm,
-            &all_constraints,
-            &inputs,
-            output_type,
-            ia.volatile,
-            ia.alignstack,
-            ia.dialect,
-            &[span],
-        );
-        if r.is_none() {
-            return false;
-        }
-        let r = r.unwrap();
-
-        // Again, based on how many outputs we have
-        let outputs = ia
-            .outputs
-            .iter()
-            .zip(&outputs)
-            .filter(|&(o, _)| !o.is_indirect);
-        for (i, (_, &place)) in outputs.enumerate() {
-            let v = if num_outputs == 1 {
-                r
-            } else {
-                self.extract_value(r, i as u64)
-            };
-            OperandValue::Immediate(v).store(self, place);
-        }
-
-        true
-    }
-
     fn codegen_inline_asm(
         &mut self,
         template: &[InlineAsmTemplatePiece],
@@ -119,6 +28,8 @@ impl<'a, 'll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         options: rustc_ast::InlineAsmOptions,
         line_spans: &[Span],
         _inst: Instance,
+        _dest: Option<Self::BasicBlock>,
+        _catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
     ) {
         // Collect the types of output operands
         let mut constraints = vec![];
@@ -223,9 +134,7 @@ impl<'a, 'll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     }
                 }
                 InlineAsmTemplatePiece::Placeholder {
-                    operand_idx,
-                    span: _,
-                    ..
+                    operand_idx, span, ..
                 } => {
                     match operands[operand_idx] {
                         InlineAsmOperandRef::In { .. }
@@ -242,6 +151,16 @@ impl<'a, 'll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                             // Only emit the raw symbol name
                             template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
                         }
+                        InlineAsmOperandRef::Label { .. } => {
+                            // template_str.push_str(&format!("${{{}:l}}", constraints.len()));
+                            // constraints.push("!i".to_owned());
+                            // labels.push(label);
+
+                            self.tcx
+                                .sess
+                                .dcx()
+                                .span_fatal(span, "Operands with label refs are unsupported");
+                        }
                     }
                 }
             }
@@ -260,7 +179,7 @@ impl<'a, 'll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             [ty] => ty,
             tys => self.type_struct(tys, false),
         };
-        let dialect = LlvmAsmDialect::Att;
+        let dialect = llvm::AsmDialect::Att;
         let result = inline_asm_call(
             self,
             &template_str,
@@ -303,7 +222,7 @@ impl<'a, 'll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
     }
 }
 
-impl<'ll, 'tcx> AsmMethods for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn codegen_global_asm(
         &self,
         template: &[InlineAsmTemplatePiece],
@@ -328,6 +247,8 @@ impl<'ll, 'tcx> AsmMethods for CodegenCx<'ll, 'tcx> {
                             // here unlike normal inline assembly.
                             template_str.push_str(string);
                         }
+                        GlobalAsmOperandRef::SymFn { .. } => todo!(),
+                        GlobalAsmOperandRef::SymStatic { .. } => todo!(),
                     }
                 }
             }
@@ -336,11 +257,15 @@ impl<'ll, 'tcx> AsmMethods for CodegenCx<'ll, 'tcx> {
         unsafe {
             llvm::LLVMRustAppendModuleInlineAsm(
                 self.llmod,
-                template_str.as_ptr().cast(),
+                template_str.as_c_char_ptr(),
                 template_str.len(),
             );
         }
     }
+
+    fn mangled_name(&self, _instance: Instance<'tcx>) -> String {
+        todo!()
+    }
 }
 
 fn reg_to_llvm(reg: InlineAsmRegOrRegClass) -> String {
@@ -367,7 +292,7 @@ pub(crate) fn inline_asm_call<'a, 'll, 'tcx>(
     output: &'ll llvm::Type,
     volatile: bool,
     alignstack: bool,
-    dia: LlvmAsmDialect,
+    dia: llvm::AsmDialect,
     line_spans: &[Span],
 ) -> Option<&'ll Value> {
     let volatile = if volatile { llvm::True } else { llvm::False };
@@ -388,9 +313,9 @@ pub(crate) fn inline_asm_call<'a, 'll, 'tcx>(
                 cons.len(),
                 volatile,
                 alignstack,
-                llvm::AsmDialect::from_generic(dia),
+                dia,
             );
-            let call = bx.call(fty, v, inputs, None);
+            let call = bx.call(fty, None, None, v, inputs, None, None);
 
             // Store mark in a metadata node so we can map LLVM errors
             // back to source locations.  See #17552.
diff --git a/crates/rustc_codegen_nvvm/src/attributes.rs b/crates/rustc_codegen_nvvm/src/attributes.rs
index 0652a0b1..5992e12a 100644
--- a/crates/rustc_codegen_nvvm/src/attributes.rs
+++ b/crates/rustc_codegen_nvvm/src/attributes.rs
@@ -1,9 +1,10 @@
 use crate::llvm::{self, AttributePlace::*, Value};
-use rustc_ast::{Attribute, Lit, LitKind};
-use rustc_attr::{InlineAttr, OptimizeAttr};
-use rustc_middle::{middle::codegen_fn_attrs::CodegenFnAttrFlags, ty};
-use rustc_session::{config::OptLevel, Session};
-use rustc_span::{sym, Symbol};
+use rustc_ast::{LitKind, MetaItemInner, MetaItemLit};
+use rustc_attr_parsing::{InlineAttr, OptimizeAttr};
+use rustc_hir::Attribute;
+use rustc_middle::{bug, middle::codegen_fn_attrs::CodegenFnAttrFlags, ty};
+use rustc_session::{Session, config::OptLevel};
+use rustc_span::{Symbol, sym};
 
 use crate::context::CodegenCx;
 
@@ -15,6 +16,7 @@ fn inline(val: &'_ Value, inline: InlineAttr) {
         Always => llvm::Attribute::AlwaysInline.apply_llfn(Function, val),
         Never => llvm::Attribute::NoInline.apply_llfn(Function, val),
         None => {}
+        Force { .. } => bug!("Force inline should have been inlined away by now"), // TODO: Verify this
     }
 }
 
@@ -49,7 +51,7 @@ pub(crate) fn from_fn_attrs<'ll, 'tcx>(
     let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id());
 
     match codegen_fn_attrs.optimize {
-        OptimizeAttr::None => {
+        OptimizeAttr::Default => {
             default_optimisation_attrs(cx.tcx.sess, llfn);
         }
         OptimizeAttr::Speed => {
@@ -62,6 +64,11 @@ pub(crate) fn from_fn_attrs<'ll, 'tcx>(
             llvm::Attribute::OptimizeForSize.apply_llfn(Function, llfn);
             llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn);
         }
+        OptimizeAttr::DoNotOptimize => {
+            llvm::Attribute::MinSize.unapply_llfn(Function, llfn);
+            llvm::Attribute::OptimizeForSize.unapply_llfn(Function, llfn);
+            llvm::Attribute::OptimizeNone.apply_llfn(Function, llfn);
+        }
     }
 
     let inline_attr = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) {
@@ -109,34 +116,23 @@ impl NvvmAttributes {
         let mut nvvm_attrs = Self::default();
 
         for attr in attrs {
-            if attr.has_name(cx.symbols.nvvm_internal) {
+            if attr.path_matches(&[cx.symbols.nvvm_internal, cx.symbols.kernel]) {
+                nvvm_attrs.kernel = true;
+            } else if attr.path_matches(&[cx.symbols.nvvm_internal, sym::used]) {
+                nvvm_attrs.used = true;
+            } else if attr.path_matches(&[cx.symbols.nvvm_internal, cx.symbols.addrspace]) {
                 let args = attr.meta_item_list().unwrap_or_default();
-                if let Some(arg) = args.first() {
-                    if arg.has_name(cx.symbols.kernel) {
-                        nvvm_attrs.kernel = true;
-                    }
-                    if arg.has_name(sym::used) {
-                        nvvm_attrs.used = true;
-                    }
-                    if arg.has_name(cx.symbols.addrspace) {
-                        let args = arg.meta_item_list().unwrap_or_default();
-                        if let Some(arg) = args.first() {
-                            let lit = arg.literal();
-                            if let Some(Lit {
-                                kind: LitKind::Int(val, _),
-                                ..
-                            }) = lit
-                            {
-                                nvvm_attrs.addrspace = Some(*val as u8);
-                            } else {
-                                panic!();
-                            }
-                        }
-                    }
+                if let Some(MetaItemInner::Lit(MetaItemLit {
+                    kind: LitKind::Int(val, _),
+                    ..
+                })) = args.first()
+                {
+                    nvvm_attrs.addrspace = Some(val.get() as u8);
+                } else {
+                    panic!();
                 }
             }
         }
-
         nvvm_attrs
     }
 }
diff --git a/crates/rustc_codegen_nvvm/src/back.rs b/crates/rustc_codegen_nvvm/src/back.rs
index 1759616d..4e80a0c1 100644
--- a/crates/rustc_codegen_nvvm/src/back.rs
+++ b/crates/rustc_codegen_nvvm/src/back.rs
@@ -1,37 +1,36 @@
-use crate::llvm::{self};
-use crate::override_fns::define_or_override_fn;
-use crate::{builder::Builder, context::CodegenCx, lto::ThinBuffer, LlvmMod, NvvmCodegenBackend};
+use std::io::{self, Write};
+use std::slice;
+use std::sync::Arc;
+
 use libc::{c_char, size_t};
 use rustc_codegen_ssa::back::write::{TargetMachineFactoryConfig, TargetMachineFactoryFn};
-use rustc_codegen_ssa::traits::{DebugInfoMethods, MiscMethods};
+use rustc_codegen_ssa::traits::{DebugInfoCodegenMethods, MiscCodegenMethods};
 use rustc_codegen_ssa::{
+    CompiledModule, ModuleCodegen,
     back::write::{CodegenContext, ModuleConfig},
     base::maybe_create_entry_wrapper,
     mono_item::MonoItemExt,
-    traits::{BaseTypeMethods, ThinBufferMethods},
-    CompiledModule, ModuleCodegen, ModuleKind,
+    traits::{BaseTypeCodegenMethods, ThinBufferMethods},
 };
-use rustc_data_structures::small_c_str::SmallCStr;
-use rustc_errors::{FatalError, Handler};
+use rustc_errors::{DiagCtxtHandle, FatalError};
 use rustc_fs_util::path_to_c_string;
 use rustc_middle::bug;
-use rustc_middle::mir::mono::MonoItem;
+use rustc_middle::mir::mono::{MonoItem, MonoItemData};
 use rustc_middle::{dep_graph, ty::TyCtxt};
-use rustc_session::config::{self, DebugInfo, OutputType};
 use rustc_session::Session;
+use rustc_session::config::{self, DebugInfo, OutputType};
 use rustc_span::Symbol;
 use rustc_target::spec::{CodeModel, RelocModel};
-use std::ffi::CString;
-use std::sync::Arc;
-use std::{
-    io::{self, Write},
-    slice,
-};
 
-pub fn llvm_err(handler: &Handler, msg: &str) -> FatalError {
+use crate::common::AsCCharPtr;
+use crate::llvm::{self};
+use crate::override_fns::define_or_override_fn;
+use crate::{LlvmMod, NvvmCodegenBackend, builder::Builder, context::CodegenCx, lto::ThinBuffer};
+
+pub fn llvm_err(handle: DiagCtxtHandle, msg: &str) -> FatalError {
     match llvm::last_error() {
-        Some(err) => handler.fatal(&format!("{}: {}", msg, err)),
-        None => handler.fatal(msg),
+        Some(err) => handle.fatal(format!("{}: {}", msg, err)),
+        None => handle.fatal(msg.to_string()),
     }
 }
 
@@ -42,7 +41,7 @@ pub fn to_llvm_opt_settings(
     match cfg {
         No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone),
         Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone),
-        Default => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
+        More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
         Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone),
         Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault),
         SizeMin => (
@@ -86,28 +85,31 @@ pub fn target_machine_factory(
 
     let ffunction_sections = sess
         .opts
-        .debugging_opts
+        .unstable_opts
         .function_sections
         .unwrap_or(sess.target.function_sections);
     let fdata_sections = ffunction_sections;
 
     let code_model = to_llvm_code_model(sess.code_model());
 
-    let triple = SmallCStr::new(&sess.target.llvm_target);
+    let triple = sess.target.llvm_target.clone();
     // let cpu = SmallCStr::new("sm_30");
-    let features = CString::new("").unwrap();
+    let features = "";
     let trap_unreachable = sess
         .opts
-        .debugging_opts
+        .unstable_opts
         .trap_unreachable
         .unwrap_or(sess.target.trap_unreachable);
 
     Arc::new(move |_config: TargetMachineFactoryConfig| {
         let tm = unsafe {
             llvm::LLVMRustCreateTargetMachine(
-                triple.as_ptr(),
+                triple.as_c_char_ptr(),
+                triple.len(),
                 std::ptr::null(),
-                features.as_ptr(),
+                0,
+                features.as_c_char_ptr(),
+                features.len(),
                 code_model,
                 reloc_model,
                 opt_level,
@@ -119,12 +121,7 @@ pub fn target_machine_factory(
                 false,
             )
         };
-        tm.ok_or_else(|| {
-            format!(
-                "Could not create LLVM TargetMachine for triple: {}",
-                triple.to_str().unwrap()
-            )
-        })
+        tm.ok_or_else(|| format!("Could not create LLVM TargetMachine for triple: {}", triple))
     })
 }
 
@@ -160,7 +157,7 @@ pub extern "C" fn demangle_callback(
 /// Compile a single module (in an nvvm context this means getting the llvm bitcode out of it)
 pub(crate) unsafe fn codegen(
     cgcx: &CodegenContext<NvvmCodegenBackend>,
-    diag_handler: &Handler,
+    dcx: DiagCtxtHandle<'_>,
     module: ModuleCodegen<LlvmMod>,
     config: &ModuleConfig,
 ) -> Result<CompiledModule, FatalError> {
@@ -177,7 +174,7 @@ pub(crate) unsafe fn codegen(
         .prof
         .generic_activity_with_arg("NVVM_module_codegen", &module.name[..]);
 
-    let llmod = module.module_llvm.llmod.as_ref().unwrap();
+    let llmod = unsafe { module.module_llvm.llmod.as_ref().unwrap() };
     let mod_name = module.name.clone();
     let module_name = Some(&mod_name[..]);
 
@@ -195,13 +192,15 @@ pub(crate) unsafe fn codegen(
         let out = cgcx
             .output_filenames
             .temp_path(OutputType::LlvmAssembly, module_name);
-        let out_c = path_to_c_string(&out);
+        let out = out.to_str().unwrap();
 
-        let result = llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback);
+        let result = unsafe {
+            llvm::LLVMRustPrintModule(llmod, out.as_c_char_ptr(), out.len(), demangle_callback)
+        };
 
         result.into_result().map_err(|()| {
-            let msg = format!("failed to write NVVM IR to {}", out.display());
-            llvm_err(diag_handler, &msg)
+            let msg = format!("failed to write NVVM IR to {}", out);
+            llvm_err(dcx, &msg)
         })?;
     }
 
@@ -219,7 +218,7 @@ pub(crate) unsafe fn codegen(
 
     if let Err(e) = std::fs::write(&out, data) {
         let msg = format!("failed to write bytecode to {}: {}", out.display(), e);
-        diag_handler.err(&msg);
+        dcx.err(msg);
     }
 
     Ok(CompiledModule {
@@ -228,6 +227,8 @@ pub(crate) unsafe fn codegen(
         object: Some(out),
         dwarf_object: None,
         bytecode: None,
+        assembly: None,
+        llvm_ir: None,
     })
 }
 
@@ -257,7 +258,15 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen
 
             let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx);
 
-            for &(mono_item, (linkage, visibility)) in &mono_items {
+            for &(
+                mono_item,
+                MonoItemData {
+                    linkage,
+                    visibility,
+                    ..
+                },
+            ) in &mono_items
+            {
                 mono_item.predefine::<Builder<'_, '_, '_>>(&cx, linkage, visibility);
             }
 
@@ -285,11 +294,14 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen
             }
 
             // Create the llvm.used and llvm.compiler.used variables.
-            if !cx.used_statics().borrow().is_empty() {
-                cx.create_used_variable();
+            if !cx.used_statics.borrow().is_empty() {
+                cx.create_used_variable_impl(c"llvm.used", &*cx.used_statics.borrow());
             }
-            if !cx.compiler_used_statics().borrow().is_empty() {
-                cx.create_compiler_used_variable();
+            if !cx.compiler_used_statics.borrow().is_empty() {
+                cx.create_used_variable_impl(
+                    c"llvm.compiler.used",
+                    &*cx.compiler_used_statics.borrow(),
+                );
             }
 
             // Finalize debuginfo
@@ -298,11 +310,7 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen
             }
         }
 
-        ModuleCodegen {
-            name: cgu_name.to_string(),
-            module_llvm: llvm_module,
-            kind: ModuleKind::Regular,
-        }
+        ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module)
     }
 
     // TODO(RDambrosio016): maybe the same cost as the llvm codegen works?
@@ -315,7 +323,7 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen
 // any big things were discovered in that timespan that we should modify.
 pub(crate) unsafe fn optimize(
     cgcx: &CodegenContext<NvvmCodegenBackend>,
-    diag_handler: &Handler,
+    diag_handler: DiagCtxtHandle<'_>,
     module: &ModuleCodegen<LlvmMod>,
     config: &ModuleConfig,
 ) -> Result<(), FatalError> {
@@ -323,7 +331,7 @@ pub(crate) unsafe fn optimize(
         .prof
         .generic_activity_with_arg("LLVM_module_optimize", &module.name[..]);
 
-    let llmod = &*module.module_llvm.llmod;
+    let llmod = unsafe { &*module.module_llvm.llmod };
 
     let module_name = module.name.clone();
     let module_name = Some(&module_name[..]);
@@ -333,65 +341,68 @@ pub(crate) unsafe fn optimize(
             .output_filenames
             .temp_path_ext("no-opt.bc", module_name);
         let out = path_to_c_string(&out);
-        llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr());
+        unsafe { llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()) };
     }
 
     let tm_factory_config = TargetMachineFactoryConfig {
         split_dwarf_file: None,
+        output_obj_file: None,
     };
 
     let tm = (cgcx.tm_factory)(tm_factory_config).expect("failed to create target machine");
 
     if config.opt_level.is_some() {
-        let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod);
-        let mpm = llvm::LLVMCreatePassManager();
-
-        let addpass = |pass_name: &str| {
-            let pass_name = CString::new(pass_name).unwrap();
-            let pass = llvm::LLVMRustFindAndCreatePass(pass_name.as_ptr());
-            if pass.is_none() {
-                return false;
-            }
-            let pass = pass.unwrap();
-            let pass_manager = match llvm::LLVMRustPassKind(pass) {
-                llvm::PassKind::Function => &fpm,
-                llvm::PassKind::Module => &mpm,
-                llvm::PassKind::Other => {
-                    diag_handler.err("Encountered LLVM pass kind we can't handle");
-                    return true;
+        unsafe {
+            let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod);
+            let mpm = llvm::LLVMCreatePassManager();
+
+            let addpass = |pass_name: &str| {
+                let pass =
+                    llvm::LLVMRustFindAndCreatePass(pass_name.as_c_char_ptr(), pass_name.len());
+                if pass.is_none() {
+                    return false;
                 }
+                let pass = pass.unwrap();
+                let pass_manager = match llvm::LLVMRustPassKind(pass) {
+                    llvm::PassKind::Function => &fpm,
+                    llvm::PassKind::Module => &mpm,
+                    llvm::PassKind::Other => {
+                        diag_handler.err("Encountered LLVM pass kind we can't handle");
+                        return true;
+                    }
+                };
+                llvm::LLVMRustAddPass(pass_manager, pass);
+                true
             };
-            llvm::LLVMRustAddPass(pass_manager, pass);
-            true
-        };
 
-        if !config.no_prepopulate_passes {
-            llvm::LLVMRustAddAnalysisPasses(tm, fpm, llmod);
-            llvm::LLVMRustAddAnalysisPasses(tm, mpm, llmod);
-            let opt_level = config
-                .opt_level
-                .map_or(llvm::CodeGenOptLevel::None, |x| to_llvm_opt_settings(x).0);
-            with_llvm_pmb(llmod, config, opt_level, &mut |b| {
-                llvm::LLVMPassManagerBuilderPopulateFunctionPassManager(b, fpm);
-                llvm::LLVMPassManagerBuilderPopulateModulePassManager(b, mpm);
-            })
-        }
+            if !config.no_prepopulate_passes {
+                llvm::LLVMRustAddAnalysisPasses(tm, fpm, llmod);
+                llvm::LLVMRustAddAnalysisPasses(tm, mpm, llmod);
+                let opt_level = config
+                    .opt_level
+                    .map_or(llvm::CodeGenOptLevel::None, |x| to_llvm_opt_settings(x).0);
+                with_llvm_pmb(llmod, config, opt_level, &mut |b| {
+                    llvm::LLVMPassManagerBuilderPopulateFunctionPassManager(b, fpm);
+                    llvm::LLVMPassManagerBuilderPopulateModulePassManager(b, mpm);
+                })
+            }
 
-        for pass in &config.passes {
-            if !addpass(pass) {
-                diag_handler.warn(&format!("unknown pass `{}`, ignoring", pass));
+            for pass in &config.passes {
+                if !addpass(pass) {
+                    diag_handler.warn(format!("unknown pass `{}`, ignoring", pass));
+                }
             }
-        }
 
-        diag_handler.abort_if_errors();
+            diag_handler.abort_if_errors();
 
-        // Finally, run the actual optimization passes
-        llvm::LLVMRustRunFunctionPassManager(fpm, llmod);
-        llvm::LLVMRunPassManager(mpm, llmod);
+            // Finally, run the actual optimization passes
+            llvm::LLVMRustRunFunctionPassManager(fpm, llmod);
+            llvm::LLVMRunPassManager(mpm, llmod);
 
-        // Deallocate managers that we're now done with
-        llvm::LLVMDisposePassManager(fpm);
-        llvm::LLVMDisposePassManager(mpm);
+            // Deallocate managers that we're now done with
+            llvm::LLVMDisposePassManager(fpm);
+            llvm::LLVMDisposePassManager(mpm);
+        }
     }
 
     Ok(())
@@ -403,64 +414,62 @@ unsafe fn with_llvm_pmb(
     opt_level: llvm::CodeGenOptLevel,
     f: &mut impl FnMut(&llvm::PassManagerBuilder),
 ) {
-    use std::ptr;
-
-    let builder = llvm::LLVMPassManagerBuilderCreate();
-    let opt_size = config
-        .opt_size
-        .map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1);
-    let inline_threshold = config.inline_threshold;
-
-    llvm::LLVMRustConfigurePassManagerBuilder(
-        builder,
-        opt_level,
-        config.merge_functions,
-        config.vectorize_slp,
-        config.vectorize_loop,
-        false,
-        ptr::null(),
-        ptr::null(),
-    );
-
-    llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32);
-
-    if opt_size != llvm::CodeGenOptSizeNone {
-        llvm::LLVMPassManagerBuilderSetDisableUnrollLoops(builder, 1);
-    }
+    unsafe {
+        use std::ptr;
+
+        let builder = llvm::LLVMPassManagerBuilderCreate();
+        let opt_size = config
+            .opt_size
+            .map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1);
+
+        llvm::LLVMRustConfigurePassManagerBuilder(
+            builder,
+            opt_level,
+            config.merge_functions,
+            config.vectorize_slp,
+            config.vectorize_loop,
+            false,
+            ptr::null(),
+            ptr::null(),
+        );
+
+        llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32);
+
+        if opt_size != llvm::CodeGenOptSizeNone {
+            llvm::LLVMPassManagerBuilderSetDisableUnrollLoops(builder, 1);
+        }
 
-    llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins);
+        llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins);
 
-    // Here we match what clang does (kinda). For O0 we only inline
-    // always-inline functions (but don't add lifetime intrinsics), at O1 we
-    // inline with lifetime intrinsics, and O2+ we add an inliner with a
-    // thresholds copied from clang.
-    match (opt_level, opt_size, inline_threshold) {
-        (.., Some(t)) => {
-            llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, t as u32);
-        }
-        (llvm::CodeGenOptLevel::Aggressive, ..) => {
-            llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 275);
-        }
-        (_, llvm::CodeGenOptSizeDefault, _) => {
-            llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 75);
-        }
-        (_, llvm::CodeGenOptSizeAggressive, _) => {
-            llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 25);
-        }
-        (llvm::CodeGenOptLevel::None, ..) => {
-            llvm::LLVMRustAddAlwaysInlinePass(builder, false);
-        }
-        (llvm::CodeGenOptLevel::Less, ..) => {
-            llvm::LLVMRustAddAlwaysInlinePass(builder, true);
-        }
-        (llvm::CodeGenOptLevel::Default, ..) => {
-            llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 225);
-        }
-        (llvm::CodeGenOptLevel::Other, ..) => {
-            bug!("CodeGenOptLevel::Other selected")
+        // Here we match what clang does (kinda). For O0 we only inline
+        // always-inline functions (but don't add lifetime intrinsics), at O1 we
+        // inline with lifetime intrinsics, and O2+ we add an inliner with a
+        // thresholds copied from clang.
+        match (opt_level, opt_size) {
+            (llvm::CodeGenOptLevel::Aggressive, ..) => {
+                llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 275);
+            }
+            (_, llvm::CodeGenOptSizeDefault) => {
+                llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 75);
+            }
+            (_, llvm::CodeGenOptSizeAggressive) => {
+                llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 25);
+            }
+            (llvm::CodeGenOptLevel::None, ..) => {
+                llvm::LLVMRustAddAlwaysInlinePass(builder, false);
+            }
+            (llvm::CodeGenOptLevel::Less, ..) => {
+                llvm::LLVMRustAddAlwaysInlinePass(builder, true);
+            }
+            (llvm::CodeGenOptLevel::Default, ..) => {
+                llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 225);
+            }
+            (llvm::CodeGenOptLevel::Other, ..) => {
+                bug!("CodeGenOptLevel::Other selected")
+            }
         }
-    }
 
-    f(builder);
-    llvm::LLVMPassManagerBuilderDispose(builder);
+        f(builder);
+        llvm::LLVMPassManagerBuilderDispose(builder);
+    }
 }
diff --git a/crates/rustc_codegen_nvvm/src/builder.rs b/crates/rustc_codegen_nvvm/src/builder.rs
index fdb0c9c6..4fe87661 100644
--- a/crates/rustc_codegen_nvvm/src/builder.rs
+++ b/crates/rustc_codegen_nvvm/src/builder.rs
@@ -1,29 +1,35 @@
-#![allow(clippy::unnecessary_mut_passed)]
+use std::borrow::Cow;
+use std::ops::Deref;
+use std::ptr;
 
-use crate::context::CodegenCx;
-use crate::int_replace::{get_transformed_type, transmute_llval};
-use crate::llvm::{self, BasicBlock, LLVMRustGetValueType, Type, Value};
-use crate::ty::LayoutLlvmExt;
 use libc::{c_char, c_uint};
+use rustc_abi as abi;
+use rustc_abi::{AddressSpace, Align, HasDataLayout, Size, TargetDataLayout, WrappingRange};
+use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::{AtomicOrdering, IntPredicate, RealPredicate, TypeKind};
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
-use rustc_codegen_ssa::MemFlags;
+use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_hir::def_id::DefId;
+use rustc_middle::bug;
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::ty::layout::{
-    FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout,
+    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
+    TyAndLayout,
 };
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
+use rustc_session::config::OptLevel;
 use rustc_span::Span;
-use rustc_target::abi::call::FnAbi;
-use rustc_target::abi::{self, AddressSpace, Align, Size, WrappingRange};
+use rustc_target::callconv::FnAbi;
 use rustc_target::spec::{HasTargetSpec, Target};
-use std::borrow::Cow;
-use std::ffi::{CStr, CString};
-use std::ops::Deref;
-use std::ptr;
-use tracing::trace;
+use tracing::{debug, trace};
+
+use crate::abi::FnAbiLlvmExt;
+use crate::context::CodegenCx;
+use crate::int_replace::{get_transformed_type, transmute_llval};
+use crate::llvm::{self, BasicBlock, Type, Value};
+use crate::ty::LayoutLlvmExt;
 
 // All Builders must have an llfn associated with them
 #[must_use]
@@ -40,15 +46,12 @@ impl<'ll, 'tcx, 'a> Drop for Builder<'a, 'll, 'tcx> {
     }
 }
 
-// cant make this a const because its not a const fn on stable yet :(
-fn empty_cstr() -> &'static CStr {
-    unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") }
-}
+const UNNAMED: *const c_char = c"".as_ptr();
 
 /// Empty string, to be used where LLVM expects an instruction name, indicating
 /// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
 pub(crate) fn unnamed() -> *const c_char {
-    empty_cstr().as_ptr()
+    UNNAMED
 }
 
 impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> {
@@ -61,10 +64,12 @@ impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> {
     type DIScope = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIScope;
     type DILocation = <CodegenCx<'ll, 'tcx> as BackendTypes>::DILocation;
     type DIVariable = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIVariable;
+
+    type Metadata = <CodegenCx<'ll, 'tcx> as BackendTypes>::Metadata;
 }
 
-impl abi::HasDataLayout for Builder<'_, '_, '_> {
-    fn data_layout(&self) -> &abi::TargetDataLayout {
+impl HasDataLayout for Builder<'_, '_, '_> {
+    fn data_layout(&self) -> &TargetDataLayout {
         self.cx.data_layout()
     }
 }
@@ -75,9 +80,9 @@ impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> {
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.cx.param_env()
+impl<'tcx> HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
+    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
+        self.cx.typing_env()
     }
 }
 
@@ -119,61 +124,41 @@ impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
     }
 }
 
-impl<'ll, 'tcx> HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> {
-    type CodegenCx = CodegenCx<'ll, 'tcx>;
-}
-
-macro_rules! builder_methods_for_value_instructions {
+macro_rules! math_builder_methods {
     ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
         $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
             unsafe {
                 trace!("binary expr: {:?} with args {:?}", stringify!($name), [$($arg),*]);
-                llvm::$llvm_capi(self.llbuilder, $($arg,)* unnamed())
+                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
             }
         })+
     }
 }
 
-impl<'a, 'll, 'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
-    fn set_function_source_hash(
-        &mut self,
-        _instance: ty::Instance<'tcx>,
-        _function_source_hash: u64,
-    ) -> bool {
-        todo!()
-    }
-
-    fn add_coverage_counter(
-        &mut self,
-        _instance: ty::Instance<'tcx>,
-        _index: rustc_middle::mir::coverage::CounterValueReference,
-        _region: rustc_middle::mir::coverage::CodeRegion,
-    ) -> bool {
-        todo!()
-    }
-
-    fn add_coverage_counter_expression(
-        &mut self,
-        _instance: ty::Instance<'tcx>,
-        _id: rustc_middle::mir::coverage::InjectedExpressionId,
-        _lhs: rustc_middle::mir::coverage::ExpressionOperandId,
-        _op: rustc_middle::mir::coverage::Op,
-        _rhs: rustc_middle::mir::coverage::ExpressionOperandId,
-        _region: Option<rustc_middle::mir::coverage::CodeRegion>,
-    ) -> bool {
-        todo!()
+macro_rules! set_math_builder_methods {
+    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
+        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
+            unsafe {
+                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
+                llvm::$llvm_set_math(instr);
+                instr
+            }
+        })+
     }
+}
 
-    fn add_coverage_unreachable(
+impl<'a, 'll, 'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
+    fn add_coverage(
         &mut self,
-        _instance: ty::Instance<'tcx>,
-        _region: rustc_middle::mir::coverage::CodeRegion,
-    ) -> bool {
-        todo!()
+        _instance: Instance<'tcx>,
+        _kind: &rustc_middle::mir::coverage::CoverageKind,
+    ) {
     }
 }
 
 impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
+    type CodegenCx = CodegenCx<'ll, 'tcx>;
+
     fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
         let bx = Builder::with_cx(cx);
         unsafe {
@@ -182,20 +167,19 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         bx
     }
 
-    fn llbb(&self) -> &'ll BasicBlock {
-        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
+    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
+        self.cx
     }
 
-    fn build_sibling_block(&mut self, name: &str) -> Self {
-        let llbb = self.append_sibling_block(name);
-        Self::build(self.cx, llbb)
+    fn llbb(&self) -> &'ll BasicBlock {
+        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
     }
 
     fn set_span(&mut self, _span: Span) {}
 
     fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
         unsafe {
-            let name = CString::new(name).unwrap();
+            let name = SmallCStr::new(name);
             llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
         }
     }
@@ -204,6 +188,10 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         Self::append_block(self.cx, self.llfn(), name)
     }
 
+    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
+        *self = Self::build(self.cx, llbb)
+    }
+
     fn ret_void(&mut self) {
         trace!("Ret void");
         unsafe {
@@ -214,10 +202,10 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn ret(&mut self, mut v: &'ll Value) {
         trace!("Ret `{:?}`", v);
         unsafe {
-            let ty = LLVMRustGetValueType(v);
+            let ty = self.val_ty(v);
             let (new_ty, changed) = get_transformed_type(self.cx, ty);
             if changed {
-                v = crate::int_replace::transmute_llval(self.llbuilder, self.cx, v, new_ty);
+                v = transmute_llval(self.llbuilder, self.cx, v, new_ty);
             }
             llvm::LLVMBuildRet(self.llbuilder, v);
         }
@@ -259,15 +247,18 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     fn invoke(
         &mut self,
-        ty: &'ll Type,
+        llty: &'ll Type,
+        fn_attrs: Option<&CodegenFnAttrs>,
+        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
         llfn: &'ll Value,
         args: &[&'ll Value],
         then: &'ll BasicBlock,
         _catch: &'ll BasicBlock,
         funclet: Option<&()>,
-    ) -> &'ll Value {
+        instance: Option<Instance<'tcx>>,
+    ) -> Self::Value {
         trace!("invoke");
-        let call = self.call(ty, llfn, args, funclet);
+        let call = self.call(llty, fn_attrs, fn_abi, llfn, args, funclet, instance);
         // exceptions arent a thing, go directly to the `then` block
         unsafe { llvm::LLVMBuildBr(self.llbuilder, then) };
         call
@@ -280,7 +271,7 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
-    builder_methods_for_value_instructions! {
+    math_builder_methods! {
         add(a, b) => LLVMBuildAdd,
         fadd(a, b) => LLVMBuildFAdd,
         sub(a, b) => LLVMBuildSub,
@@ -312,44 +303,17 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         unchecked_umul(x, y) => LLVMBuildNUWMul,
     }
 
-    fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, unnamed());
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
-    }
-
-    fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, unnamed());
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
-    }
-
-    fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, unnamed());
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
-    }
-
-    fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, unnamed());
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
-    }
-
-    fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, unnamed());
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
+    set_math_builder_methods! {
+        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
+        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
+        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
+        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
+        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
+        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
+        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
+        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
+        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
+        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
     }
 
     fn checked_binop(
@@ -361,9 +325,7 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     ) -> (Self::Value, Self::Value) {
         trace!(
             "Checked binop `{:?}`, lhs: `{:?}`, rhs: `{:?}`",
-            ty,
-            lhs,
-            rhs
+            ty, lhs, rhs
         );
         use rustc_middle::ty::IntTy::*;
         use rustc_middle::ty::UintTy::*;
@@ -423,10 +385,8 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             },
         };
 
-        let intrinsic = self.get_intrinsic(name);
-        // call actually ignores the ty param for now, we just need it for conformance with nightly api
-        // so give it a dummy type
-        let res = self.call(self.type_i1(), intrinsic, &[lhs, rhs], None);
+        let (ty, f) = self.get_intrinsic(name);
+        let res = self.call(ty, None, None, f, &[lhs, rhs], None, None);
         (self.extract_value(res, 0), self.extract_value(res, 1))
     }
 
@@ -444,43 +404,45 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         val
     }
 
-    fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value {
-        trace!("Alloca `{:?}`", ty);
+    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
+        trace!("Alloca `{:?}`", size);
         let mut bx = Builder::with_cx(self.cx);
         bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
-        bx.dynamic_alloca(ty, align)
-    }
-
-    fn dynamic_alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value {
-        trace!("Dynamic Alloca `{:?}`", ty);
+        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
         unsafe {
-            let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, unnamed());
+            let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, ty, UNNAMED);
             llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
-            alloca
+            // Cast to default addrspace if necessary
+            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
         }
     }
 
-    fn array_alloca(&mut self, ty: &'ll Type, len: &'ll Value, align: Align) -> &'ll Value {
+    fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value {
+        trace!("Dynamic Alloca `{:?}`", size);
         unsafe {
-            let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, ty, len, unnamed());
+            let alloca =
+                llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED);
             llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
-            alloca
+            // Cast to default addrspace if necessary
+            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
         }
     }
 
-    fn load(&mut self, _: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
-        trace!("Load `{:?}`", ptr);
+    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
+        trace!("Load {ty:?} {:?}", ptr);
+        let ptr = self.pointercast(ptr, self.cx.type_ptr_to(ty));
         unsafe {
-            let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, unnamed());
+            let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
             llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
             load
         }
     }
 
-    fn volatile_load(&mut self, _: &'ll Type, ptr: &'ll Value) -> &'ll Value {
+    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
         trace!("Volatile load `{:?}`", ptr);
+        let ptr = self.pointercast(ptr, self.cx.type_ptr_to(ty));
         unsafe {
-            let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, unnamed());
+            let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
             llvm::LLVMSetVolatile(load, llvm::True);
             load
         }
@@ -488,7 +450,7 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     fn atomic_load(
         &mut self,
-        ty: &'ll Type,
+        _ty: &'ll Type,
         ptr: &'ll Value,
         _order: AtomicOrdering,
         _size: Size,
@@ -509,77 +471,122 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
         // self.call(vprintf, &[formatlist, valist], None);
 
-        let trap = self.get_intrinsic("llvm.trap");
-        self.call(ty, trap, &[], None);
+        let (ty, f) = self.get_intrinsic("llvm.trap");
+        self.call(ty, None, None, f, &[], None, None);
         unsafe { llvm::LLVMBuildLoad(self.llbuilder, ptr, unnamed()) }
     }
 
     fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
-        trace!("Load operand `{:?}`", place);
-        assert_eq!(place.llextra.is_some(), place.layout.is_unsized());
+        if place.layout.is_unsized() {
+            let tail = self
+                .tcx
+                .struct_tail_for_codegen(place.layout.ty, self.typing_env());
+            if matches!(tail.kind(), ty::Foreign(..)) {
+                // Unsized locals and, at least conceptually, even unsized arguments must be copied
+                // around, which requires dynamically determining their size. Therefore, we cannot
+                // allow `extern` types here. Consult t-opsem before removing this check.
+                panic!("unsized locals must not be `extern` types");
+            }
+        }
+        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
 
         if place.layout.is_zst() {
-            return OperandRef::new_zst(self, place.layout);
+            return OperandRef::zero_sized(place.layout);
         }
 
         fn scalar_load_metadata<'a, 'll, 'tcx>(
             bx: &mut Builder<'a, 'll, 'tcx>,
             load: &'ll Value,
-            scalar: &abi::Scalar,
+            scalar: abi::Scalar,
+            layout: TyAndLayout<'tcx>,
+            offset: Size,
         ) {
-            match scalar.value {
-                abi::Int(..) => {
+            if bx.sess().opts.optimize == OptLevel::No {
+                // Don't emit metadata we're not going to use
+                return;
+            }
+
+            if !scalar.is_uninit_valid() {
+                bx.noundef_metadata(load);
+            }
+
+            match scalar.primitive() {
+                abi::Primitive::Int(..) => {
                     if !scalar.is_always_valid(bx) {
-                        bx.range_metadata(load, scalar.valid_range);
+                        bx.range_metadata(load, scalar.valid_range(bx));
                     }
                 }
-                abi::Pointer if !scalar.valid_range.contains(0) => {
-                    bx.nonnull_metadata(load);
+                abi::Primitive::Pointer(_) => {
+                    if !scalar.valid_range(bx).contains(0) {
+                        bx.nonnull_metadata(load);
+                    }
+
+                    if let Some(pointee) = layout.pointee_info_at(bx, offset) {
+                        if let Some(_) = pointee.safe {
+                            bx.align_metadata(load, pointee.align);
+                        }
+                    }
                 }
-                _ => {}
+                abi::Primitive::Float(_) => {}
             }
         }
 
-        let val = if let Some(llextra) = place.llextra {
-            OperandValue::Ref(place.llval, Some(llextra), place.align)
+        let val = if let Some(_) = place.val.llextra {
+            // FIXME: Merge with the `else` below?
+            OperandValue::Ref(place.val)
         } else if place.layout.is_llvm_immediate() {
             let mut const_llval = None;
+            let llty = place.layout.llvm_type(self);
             unsafe {
-                if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) {
+                if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
                     if llvm::LLVMIsGlobalConstant(global) == llvm::True {
-                        const_llval = llvm::LLVMGetInitializer(global);
+                        if let Some(init) = llvm::LLVMGetInitializer(global) {
+                            if self.val_ty(init) == llty {
+                                const_llval = Some(init);
+                            }
+                        }
                     }
                 }
             }
 
             let llval = const_llval.unwrap_or_else(|| {
-                let load = self.load(place.layout.llvm_type(self), place.llval, place.align);
-
-                if let abi::Abi::Scalar(ref scalar) = place.layout.abi {
-                    scalar_load_metadata(self, load, scalar);
+                let load = self.load(llty, place.val.llval, place.val.align);
+                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
+                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
+                    self.to_immediate_scalar(load, scalar)
+                } else {
+                    load
                 }
-                load
             });
 
-            OperandValue::Immediate(self.to_immediate(llval, place.layout))
-        } else if let abi::Abi::ScalarPair(ref a, ref b) = place.layout.abi {
-            let b_offset = a.value.size(self).align_to(b.value.align(self).abi);
-            let pair_ty = place.layout.llvm_type(self);
+            OperandValue::Immediate(llval)
+        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
+            let b_offset = a.size(self).align_to(b.align(self).abi);
 
-            let mut load = |i, scalar: &abi::Scalar, align| {
-                let llptr = self.struct_gep(pair_ty, place.llval, i as u64);
+            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
+                let llptr = if i == 0 {
+                    place.val.llval
+                } else {
+                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
+                };
                 let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
                 let load = self.load(llty, llptr, align);
-                scalar_load_metadata(self, load, scalar);
-                self.to_immediate_scalar(load, *scalar)
+                scalar_load_metadata(self, load, scalar, layout, offset);
+                self.to_immediate_scalar(load, scalar)
             };
 
             OperandValue::Pair(
-                load(0, a, place.align),
-                load(1, b, place.align.restrict_for_offset(b_offset)),
+                load(0, a, place.layout, place.val.align, Size::ZERO),
+                load(
+                    1,
+                    b,
+                    place.layout,
+                    place.val.align.restrict_for_offset(b_offset),
+                    b_offset,
+                ),
             )
         } else {
-            OperandValue::Ref(place.llval, None, place.align)
+            OperandValue::Ref(place.val)
         };
 
         OperandRef {
@@ -589,48 +596,40 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn write_operand_repeatedly(
-        mut self,
+        &mut self,
         cg_elem: OperandRef<'tcx, &'ll Value>,
         count: u64,
         dest: PlaceRef<'tcx, &'ll Value>,
-    ) -> Self {
+    ) {
         trace!("write operand repeatedly");
         let zero = self.const_usize(0);
         let count = self.const_usize(count);
-        let start = dest.project_index(&mut self, zero).llval;
-        let end = dest.project_index(&mut self, count).llval;
 
-        let mut header_bx = self.build_sibling_block("repeat_loop_header");
-        let mut body_bx = self.build_sibling_block("repeat_loop_body");
-        let next_bx = self.build_sibling_block("repeat_loop_next");
+        let header_bb = self.append_sibling_block("repeat_loop_header");
+        let body_bb = self.append_sibling_block("repeat_loop_body");
+        let next_bb = self.append_sibling_block("repeat_loop_next");
 
-        self.br(header_bx.llbb());
-        let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]);
+        self.br(header_bb);
 
-        let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end);
-        header_bx.cond_br(keep_going, body_bx.llbb(), next_bx.llbb());
+        let mut header_bx = Self::build(self.cx, header_bb);
+        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
 
-        let align = dest
-            .align
-            .restrict_for_offset(dest.layout.field(self.cx(), 0).size);
-        cg_elem.val.store(
-            &mut body_bx,
-            PlaceRef::new_sized_aligned(current, cg_elem.layout, align),
-        );
+        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
+        header_bx.cond_br(keep_going, body_bb, next_bb);
 
-        let next = body_bx.inbounds_gep(
-            self.backend_type(cg_elem.layout),
-            current,
-            &[self.const_usize(1)],
-        );
-        body_bx.br(header_bx.llbb());
-        header_bx.add_incoming_to_phi(current, next, body_bx.llbb());
+        let mut body_bx = Self::build(self.cx, body_bb);
+        let dest_elem = dest.project_index(&mut body_bx, i);
+        cg_elem.val.store(&mut body_bx, dest_elem);
+
+        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
+        body_bx.br(header_bb);
+        header_bx.add_incoming_to_phi(i, next, body_bb);
 
-        next_bx
+        *self = Self::build(self.cx, next_bb);
     }
 
     fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
-        trace!("range metadata on {:?}: {:?}", load, range);
+        trace!("range metadata on {load:?}: {range:?}");
         unsafe {
             let llty = self.cx.val_ty(load);
             let v = [
@@ -656,15 +655,6 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
-    fn type_metadata(&mut self, _function: &'ll Value, _typeid: String) {
-        // LLVM CFI doesnt make sense on the GPU
-    }
-
-    fn typeid_metadata(&mut self, _typeid: String) -> Self::Value {
-        // LLVM CFI doesnt make sense on the GPU
-        self.const_i32(0)
-    }
-
     fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
         trace!("Store val `{:?}` into ptr `{:?}`", val, ptr);
         self.store_with_flags(val, ptr, align, MemFlags::empty())
@@ -677,6 +667,13 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         align: Align,
         flags: MemFlags,
     ) -> &'ll Value {
+        trace!(
+            "store_with_flags: {:?} into {:?} with align {:?}",
+            val,
+            ptr,
+            align.bytes()
+        );
+        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
         let ptr = self.check_store(val, ptr);
         unsafe {
             let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
@@ -721,124 +718,131 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         // self.call(vprintf, &[formatlist, valist], None);
         self.abort();
         unsafe {
-            llvm::LLVMBuildLoad(self.llbuilder, ptr, unnamed());
+            llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
         }
     }
 
-    fn gep(&mut self, _: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
-        trace!("gep: {:?} with indices {:?}", ptr, indices);
+    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
+        trace!("gep: {ty:?} {:?} with indices {:?}", ptr, indices);
+        let ptr = self.pointercast(ptr, self.cx().type_ptr_to(ty));
         unsafe {
-            llvm::LLVMBuildGEP(
+            llvm::LLVMBuildGEP2(
                 self.llbuilder,
+                ty,
                 ptr,
                 indices.as_ptr(),
                 indices.len() as c_uint,
-                unnamed(),
+                UNNAMED,
             )
         }
     }
 
     fn inbounds_gep(
         &mut self,
-        _: &'ll Type,
+        ty: &'ll Type,
         ptr: &'ll Value,
         indices: &[&'ll Value],
     ) -> &'ll Value {
-        trace!("gep inbounds: {:?} with indices {:?}", ptr, indices);
+        trace!("gep inbounds: {ty:?} {:?} with indices {:?}", ptr, indices);
+        let ptr = self.pointercast(ptr, self.cx().type_ptr_to(ty));
         unsafe {
-            llvm::LLVMBuildInBoundsGEP(
+            llvm::LLVMBuildInBoundsGEP2(
                 self.llbuilder,
+                ty,
                 ptr,
                 indices.as_ptr(),
                 indices.len() as c_uint,
-                unnamed(),
+                UNNAMED,
             )
         }
     }
 
-    fn struct_gep(&mut self, _: &'ll Type, ptr: &'ll Value, idx: u64) -> &'ll Value {
-        trace!("struct gep: {:?}, {:?}", ptr, idx);
-        assert_eq!(idx as c_uint as u64, idx);
-        unsafe { llvm::LLVMBuildStructGEP(self.llbuilder, ptr, idx as c_uint, unnamed()) }
-    }
-
     /* Casts */
     fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("trunc {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
     fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("sext {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
-    fn fptoui_sat(&mut self, _val: &'ll Value, _dest_ty: &'ll Type) -> Option<&'ll Value> {
-        None
+    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+        // NVVM does not have support for saturated conversion. Setting rustc flag
+        // `-Z saturating_float_casts=false` falls back to non-saturated, UB-prone
+        // conversion, and should prevent this codegen. Otherwise, fall back to UB
+        // prone conversion.
+        self.cx().sess().dcx()
+            .warn("Saturated float to int conversion is not supported on NVVM. Defaulting to UB prone conversion.");
+        self.fptoui(val, dest_ty)
     }
 
-    fn fptosi_sat(&mut self, _val: &'ll Value, _dest_ty: &'ll Type) -> Option<&'ll Value> {
-        None
+    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+        // NVVM does not have support for saturated conversion. Setting rustc flag
+        // `-Z saturating_float_casts=false` falls back to non-saturated, UB-prone
+        // conversion, and should prevent this codegen. Otherwise, fall back to UB
+        // prone conversion.
+        self.cx().sess().dcx()
+            .warn("Saturated float to int conversion is not supported on NVVM. Defaulting to UB prone conversion.");
+        self.fptosi(val, dest_ty)
     }
 
-    // fn fptosui_may_trap(&self, val: &'ll Value, dest_ty: &'ll Type) -> bool {
-    //     false
-    // }
-
     fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("fptoui {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
     fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("fptosi {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
     fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("uitofp {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
     fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("sitofp {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
     fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("fptrunc {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
     fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("fpext {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
     fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("ptrtoint {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
     fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("inttoptr {:?} to {:?}", val, dest_ty);
-        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, unnamed()) }
+        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
-    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+    fn bitcast(&mut self, mut val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         trace!("Bitcast `{:?}` to ty `{:?}`", val, dest_ty);
         unsafe {
-            let ty = llvm::LLVMRustGetValueType(val);
+            let ty = self.val_ty(val);
             let kind = llvm::LLVMRustGetTypeKind(ty);
             if kind == llvm::TypeKind::Pointer {
-                let element = llvm::LLVMGetElementType(ty);
+                let element = self.element_type(ty);
                 let addrspace = llvm::LLVMGetPointerAddressSpace(ty);
                 let new_ty = self.type_ptr_to_ext(element, AddressSpace::DATA);
                 if addrspace != 0 {
-                    return llvm::LLVMBuildAddrSpaceCast(self.llbuilder, val, new_ty, unnamed());
+                    trace!("injecting addrspace cast for `{:?}` to `{:?}`", ty, new_ty);
+                    val = llvm::LLVMBuildAddrSpaceCast(self.llbuilder, val, new_ty, UNNAMED);
                 }
             }
-            llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, unnamed())
+            llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED)
         }
     }
 
@@ -882,8 +886,6 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         );
         let size = self.intcast(size, self.type_isize(), false);
         let is_volatile = flags.contains(MemFlags::VOLATILE);
-        let dst = self.pointercast(dst, self.type_i8p());
-        let src = self.pointercast(src, self.type_i8p());
         unsafe {
             llvm::LLVMRustBuildMemCpy(
                 self.llbuilder,
@@ -912,8 +914,6 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         );
         let size = self.intcast(size, self.type_isize(), false);
         let is_volatile = flags.contains(MemFlags::VOLATILE);
-        let dst = self.pointercast(dst, self.type_i8p());
-        let src = self.pointercast(src, self.type_i8p());
         unsafe {
             llvm::LLVMRustBuildMemMove(
                 self.llbuilder,
@@ -936,7 +936,6 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         flags: MemFlags,
     ) {
         let is_volatile = flags.contains(MemFlags::VOLATILE);
-        let ptr = self.pointercast(ptr, self.type_i8p());
         unsafe {
             llvm::LLVMRustBuildMemSet(
                 self.llbuilder,
@@ -956,7 +955,7 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         else_val: &'ll Value,
     ) -> &'ll Value {
         unsafe {
-            if llvm::LLVMRustGetValueType(cond) == llvm::LLVMVectorType(self.type_i1(), 2) {
+            if self.val_ty(cond) == llvm::LLVMVectorType(self.type_i1(), 2) {
                 cond = self.const_bool(false);
             }
             llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, unnamed())
@@ -982,35 +981,47 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, unnamed()) }
     }
 
-    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
-        trace!("insert value {:?}, {:?}", agg_val, idx);
+    fn insert_value(&mut self, agg_val: &'ll Value, mut elt: &'ll Value, idx: u64) -> &'ll Value {
+        trace!("insert value {:?}, {:?}, {:?}", agg_val, elt, idx);
         assert_eq!(idx as c_uint as u64, idx);
-        unsafe {
-            llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, unnamed())
+
+        let elt_ty = self.cx.val_ty(elt);
+        if self.cx.type_kind(elt_ty) == TypeKind::Pointer {
+            let agg_ty = self.cx.val_ty(agg_val);
+            let idx_ty = match self.cx.type_kind(agg_ty) {
+                TypeKind::Struct => unsafe {
+                    llvm::LLVMStructGetTypeAtIndex(agg_ty, idx as c_uint)
+                },
+                TypeKind::Array => unsafe { llvm::LLVMGetElementType(agg_ty) },
+                _ => bug!(
+                    "insert_value: expected struct or array type, found {:?}",
+                    self.cx.type_kind(agg_ty)
+                ),
+            };
+            assert_eq!(self.cx.type_kind(idx_ty), TypeKind::Pointer);
+            if idx_ty != elt_ty {
+                elt = self.pointercast(elt, idx_ty);
+            }
         }
+
+        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
     }
 
-    fn landing_pad(
-        &mut self,
-        _ty: &'ll Type,
-        _pers_fn: &'ll Value,
-        _num_clauses: usize,
-    ) -> &'ll Value {
-        todo!();
+    fn cleanup_landing_pad(&mut self, _pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
+        todo!()
     }
 
-    fn set_cleanup(&mut self, _landing_pad: &'ll Value) {}
+    fn filter_landing_pad(&mut self, _pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
+        todo!()
+    }
 
-    fn resume(&mut self, _exn: &'ll Value) -> &'ll Value {
+    fn resume(&mut self, _exn0: &'ll Value, _exn1: &'ll Value) {
         self.unsupported("resumes");
     }
 
     fn cleanup_pad(&mut self, _parent: Option<&'ll Value>, _args: &[&'ll Value]) {}
 
-    fn cleanup_ret(&mut self, _funclet: &(), _unwind: Option<&'ll BasicBlock>) -> &'ll Value {
-        // rustc doesnt actually use this value ;)
-        self.const_bool(false)
-    }
+    fn cleanup_ret(&mut self, _funclet: &(), _unwind: Option<&'ll BasicBlock>) {}
 
     fn catch_pad(&mut self, _parent: &'ll Value, _args: &[&'ll Value]) {}
 
@@ -1018,15 +1029,11 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         &mut self,
         _parent: Option<&'ll Value>,
         _unwind: Option<&'ll BasicBlock>,
-        _num_handlers: usize,
+        _handlers: &[&'ll BasicBlock],
     ) -> &'ll Value {
         self.unsupported("catch switches");
     }
 
-    fn add_handler(&mut self, _catch_switch: &'ll Value, _handler: &'ll BasicBlock) {
-        todo!();
-    }
-
     fn set_personality_fn(&mut self, _personality: &'ll Value) {}
 
     // Atomic Operations
@@ -1038,7 +1045,7 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         _order: rustc_codegen_ssa::common::AtomicOrdering,
         _failure_order: rustc_codegen_ssa::common::AtomicOrdering,
         _weak: bool,
-    ) -> &'ll Value {
+    ) -> (&'ll Value, &'ll Value) {
         // allowed but only for some things and with restrictions
         // https://docs.nvidia.com/cuda/nvvm-ir-spec/index.html#cmpxchg-instruction
         self.fatal("atomic cmpxchg is not supported")
@@ -1080,49 +1087,45 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
     }
 
-    fn instrprof_increment(
-        &mut self,
-        _fn_name: &'ll Value,
-        _hash: &'ll Value,
-        _num_counters: &'ll Value,
-        _index: &'ll Value,
-    ) {
-        todo!();
-    }
-
     fn call(
         &mut self,
-        _: &'ll Type,
+        llty: &'ll Type,
+        _fn_attrs: Option<&CodegenFnAttrs>,
+        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
         llfn: &'ll Value,
         args: &[&'ll Value],
-        _funclet: Option<&()>,
+        _funclet: Option<&Self::Funclet>,
+        _instance: Option<Instance<'tcx>>,
     ) -> &'ll Value {
         trace!("Calling fn {:?} with args {:?}", llfn, args);
         self.cx.last_call_llfn.set(None);
-        let args = self.check_call("call", llfn, args);
+        let args = self.check_call("call", llty, llfn, args);
 
-        let mut ret = unsafe {
+        let mut call = unsafe {
             llvm::LLVMRustBuildCall(
                 self.llbuilder,
                 llfn,
-                args.as_ptr() as *const &llvm::Value,
+                args.as_ptr(),
                 args.len() as c_uint,
                 None,
             )
         };
+        if let Some(fn_abi) = fn_abi {
+            fn_abi.apply_attrs_callsite(self, call);
+        }
 
         // bitcast return type if the type was remapped
         let map = self.cx.remapped_integer_args.borrow();
-        let mut fn_ty = unsafe { LLVMRustGetValueType(llfn) };
+        let mut fn_ty = self.val_ty(llfn);
         while self.cx.type_kind(fn_ty) == TypeKind::Pointer {
             fn_ty = self.cx.element_type(fn_ty);
         }
         if let Some((Some(ret_ty), _)) = map.get(fn_ty) {
-            self.cx.last_call_llfn.set(Some(ret));
-            ret = transmute_llval(self.llbuilder, self.cx, ret, ret_ty);
+            self.cx.last_call_llfn.set(Some(call));
+            call = transmute_llval(self.llbuilder, self.cx, call, ret_ty);
         }
 
-        ret
+        call
     }
 
     fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
@@ -1130,20 +1133,30 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, unnamed()) }
     }
 
-    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
-        self.cx
-    }
+    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
+        // Cleanup is always the cold path.
+        llvm::Attribute::Cold.apply_callsite(llvm::AttributePlace::Function, llret);
 
-    fn do_not_inline(&mut self, llret: &'ll Value) {
+        // In LLVM versions with deferred inlining (currently, system LLVM < 14),
+        // inlining drop glue can lead to exponential size blowup.
+        // See rust_lang/rust #41696 and #92110.
         llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret);
     }
+
+    fn assume_nonnull(&mut self, val: Self::Value) {
+        assert_eq!(self.cx.type_kind(self.cx.val_ty(val)), TypeKind::Pointer);
+        let val_ty = self.cx.val_ty(val);
+        let null = self.cx.const_null(val_ty);
+        let is_null = self.icmp(IntPredicate::IntNE, val, null);
+        self.assume(is_null);
+    }
 }
 
 impl<'a, 'll, 'tcx> StaticBuilderMethods for Builder<'a, 'll, 'tcx> {
     fn get_static(&mut self, def_id: DefId) -> &'ll Value {
         unsafe {
             let mut g = self.cx.get_static(def_id);
-            let llty = llvm::LLVMRustGetValueType(g);
+            let llty = self.val_ty(g);
             let addrspace = AddressSpace(llvm::LLVMGetPointerAddressSpace(llty));
             if addrspace != AddressSpace::DATA {
                 trace!("Remapping global address space of global {:?}", g);
@@ -1182,6 +1195,10 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         }
     }
 
+    fn align_metadata(&mut self, _load: &'ll Value, _align: Align) {}
+
+    fn noundef_metadata(&mut self, _load: &'ll Value) {}
+
     fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value {
         let dest_ptr_ty = self.cx.val_ty(ptr);
         let stored_ty = self.cx.val_ty(val);
@@ -1199,15 +1216,10 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
     fn check_call<'b>(
         &mut self,
         typ: &str,
+        fn_ty: &'ll Type,
         llfn: &'ll Value,
         args: &'b [&'ll Value],
     ) -> Cow<'b, [&'ll Value]> {
-        let mut fn_ty = self.cx.val_ty(llfn);
-        // Strip off pointers
-        while self.cx.type_kind(fn_ty) == TypeKind::Pointer {
-            fn_ty = self.cx.element_type(fn_ty);
-        }
-
         assert!(
             self.cx.type_kind(fn_ty) == TypeKind::Function,
             "builder::{} not passed a function, but {:?}",
@@ -1230,14 +1242,13 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             .into_iter()
             .zip(args.iter())
             .enumerate()
-            .map(|(_, (expected_ty, &actual_val))| {
+            .map(|(i, (expected_ty, &actual_val))| {
                 let actual_ty = self.val_ty(actual_val);
-
                 if expected_ty != actual_ty {
-                    trace!(
-                        "Expected arg to be {:?} but instead found {:?}, bitcasting the pain away",
-                        expected_ty,
-                        actual_ty
+                    debug!(
+                        "type mismatch in function call of {:?}. \
+                            Expected {:?} for param {}, got {:?}; injecting bitcast",
+                        llfn, expected_ty, i, actual_ty
                     );
                     self.bitcast(actual_val, expected_ty)
                 } else {
@@ -1253,6 +1264,11 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, unnamed()) }
     }
 
+    pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value {
+        let (ty, f) = self.cx.get_intrinsic(intrinsic);
+        self.call(ty, None, None, f, args, None, None)
+    }
+
     fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
         let size = size.bytes();
         if size == 0 {
@@ -1263,15 +1279,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             return;
         }
 
-        let lifetime_intrinsic = self.cx.get_intrinsic(intrinsic);
-
-        let ptr = self.pointercast(ptr, self.cx.type_i8p());
-        self.call(
-            self.type_i1(),
-            lifetime_intrinsic,
-            &[self.cx.const_u64(size), ptr],
-            None,
-        );
+        self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]);
     }
 
     pub(crate) fn phi(
diff --git a/crates/rustc_codegen_nvvm/src/common.rs b/crates/rustc_codegen_nvvm/src/common.rs
new file mode 100644
index 00000000..e437aebb
--- /dev/null
+++ b/crates/rustc_codegen_nvvm/src/common.rs
@@ -0,0 +1,19 @@
+use libc::c_char;
+
+/// Extension trait for explicit casts to `*const c_char`.
+pub(crate) trait AsCCharPtr {
+    /// Equivalent to `self.as_ptr().cast()`, but only casts to `*const c_char`.
+    fn as_c_char_ptr(&self) -> *const c_char;
+}
+
+impl AsCCharPtr for str {
+    fn as_c_char_ptr(&self) -> *const c_char {
+        self.as_ptr().cast()
+    }
+}
+
+impl AsCCharPtr for [u8] {
+    fn as_c_char_ptr(&self) -> *const c_char {
+        self.as_ptr().cast()
+    }
+}
diff --git a/crates/rustc_codegen_nvvm/src/const_ty.rs b/crates/rustc_codegen_nvvm/src/const_ty.rs
index 96d6dcb6..a03de2a2 100644
--- a/crates/rustc_codegen_nvvm/src/const_ty.rs
+++ b/crates/rustc_codegen_nvvm/src/const_ty.rs
@@ -1,22 +1,22 @@
 use crate::llvm::{self, Bool, False, True, Type, Value};
 use crate::{consts::const_alloc_to_llvm, context::CodegenCx, ty::LayoutLlvmExt};
-use abi::Primitive::Pointer;
 use libc::c_uint;
+use rustc_abi as abi;
+use rustc_abi::Primitive::Pointer;
+use rustc_abi::{AddressSpace, HasDataLayout};
 use rustc_ast::Mutability;
-use rustc_codegen_ssa::{
-    mir::place::PlaceRef,
-    traits::{BaseTypeMethods, ConstMethods, DerivedTypeMethods, MiscMethods, StaticMethods},
-};
-use rustc_middle::mir::interpret::{Allocation, GlobalAlloc, Scalar};
+use rustc_codegen_ssa::common::TypeKind;
+use rustc_codegen_ssa::traits::*;
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_hashes::Hash128;
+use rustc_middle::bug;
+use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
 use rustc_middle::ty::layout::LayoutOf;
-use rustc_middle::ty::{layout::TyAndLayout, ScalarInt};
-use rustc_span::Symbol;
-use rustc_target::abi::{self, AddressSpace, HasDataLayout, Size};
 use tracing::trace;
 
-impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
-    fn const_data_from_alloc(&self, alloc: &Allocation) -> &'ll Value {
-        const_alloc_to_llvm(self, alloc)
+impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+    fn const_data_from_alloc(&self, alloc: ConstAllocation) -> &'ll Value {
+        const_alloc_to_llvm(self, alloc, /*static*/ false)
     }
 
     fn const_null(&self, t: &'ll Type) -> &'ll Value {
@@ -32,10 +32,18 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     }
 
     fn const_uint(&self, t: &'ll Type, i: u64) -> &'ll Value {
+        debug_assert!(
+            self.type_kind(t) == TypeKind::Integer,
+            "only allows integer types in const_int"
+        );
         unsafe { llvm::LLVMConstInt(t, i, False) }
     }
 
     fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value {
+        debug_assert!(
+            self.type_kind(t) == TypeKind::Integer,
+            "only allows integer types in const_uint_big"
+        );
         unsafe {
             let words = [u as u64, (u >> 64) as u64];
             llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr())
@@ -70,9 +78,29 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         unsafe { llvm::LLVMConstReal(t, val) }
     }
 
-    fn const_str(&self, s: Symbol) -> (&'ll Value, &'ll Value) {
-        let len = s.as_str().len();
-        let val = self.const_cstr(s, false);
+    fn const_str(&self, s: &str) -> (&'ll Value, &'ll Value) {
+        let val = *self
+            .const_cstr_cache
+            .borrow_mut()
+            .raw_entry_mut()
+            .from_key(s)
+            .or_insert_with(|| {
+                let sc = self.const_bytes(s.as_bytes());
+                let sym = self.generate_local_symbol_name("str");
+                let g = self
+                    .define_global(&sym[..], self.val_ty(sc), AddressSpace::DATA)
+                    .unwrap_or_else(|| {
+                        bug!("symbol `{}` is already defined", sym);
+                    });
+                unsafe {
+                    llvm::LLVMSetInitializer(g, sc);
+                    llvm::LLVMSetGlobalConstant(g, True);
+                    llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage);
+                }
+                (s.to_owned(), g)
+            })
+            .1;
+        let len = s.len();
         let ty = self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self));
         let cs = unsafe { llvm::LLVMConstPointerCast(val, ty) };
         (cs, self.const_usize(len as u64))
@@ -113,59 +141,93 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         let bitsize = if layout.is_bool() {
             1
         } else {
-            layout.value.size(self).bits()
+            layout.size(self).bits()
         };
         let val = match cv {
-            Scalar::Int(ScalarInt::ZST) => {
-                assert_eq!(0, layout.value.size(self).bytes());
-                self.const_undef(self.type_ix(0))
-            }
             Scalar::Int(int) => {
-                let data = int.assert_bits(layout.value.size(self));
+                let data = int.to_bits(layout.size(self));
                 let llval = self.const_uint_big(self.type_ix(bitsize), data);
-                if layout.value == Pointer {
+                if matches!(layout.primitive(), abi::Primitive::Pointer(_)) {
                     unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
                 } else {
                     self.const_bitcast(llval, llty)
                 }
             }
-            Scalar::Ptr(ptr, _size) => {
-                let (alloc_id, offset) = ptr.into_parts();
-                let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) {
+            Scalar::Ptr(ptr, _) => {
+                let (prov, offset) = ptr.into_parts();
+                let (base_addr, base_addr_space) = match self.tcx.global_alloc(prov.alloc_id()) {
                     GlobalAlloc::Memory(alloc) => {
-                        let init = const_alloc_to_llvm(self, alloc);
-                        let value = match alloc.mutability {
-                            Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
-                            _ => self.static_addr_of(init, alloc.align, None),
-                        };
-                        if !self.sess().fewer_names() {
-                            llvm::set_value_name(value, format!("{:?}", alloc_id).as_bytes());
+                        // For ZSTs directly codegen an aligned pointer.
+                        // This avoids generating a zero-sized constant value and actually needing a
+                        // real address at runtime.
+                        if alloc.inner().len() == 0 {
+                            assert_eq!(offset.bytes(), 0);
+                            let llval = self.const_usize(alloc.inner().align.bytes());
+                            return if matches!(layout.primitive(), abi::Primitive::Pointer(_)) {
+                                unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
+                            } else {
+                                self.const_bitcast(llval, llty)
+                            };
+                        } else {
+                            let init = const_alloc_to_llvm(self, alloc, /*static*/ false);
+                            let alloc = alloc.inner();
+                            let value = match alloc.mutability {
+                                Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
+                                _ => self.static_addr_of(init, alloc.align, None),
+                            };
+                            if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty()
+                            {
+                                let hash = self.tcx.with_stable_hashing_context(|mut hcx| {
+                                    let mut hasher = StableHasher::new();
+                                    alloc.hash_stable(&mut hcx, &mut hasher);
+                                    hasher.finish::<Hash128>()
+                                });
+                                llvm::set_value_name(
+                                    value,
+                                    format!("alloc_{hash:032x}").as_bytes(),
+                                );
+                            }
+                            (value, AddressSpace::DATA)
                         }
-                        (value, AddressSpace::DATA)
                     }
-                    GlobalAlloc::Function(fn_instance) => (
-                        self.get_fn_addr(fn_instance.polymorphize(self.tcx)),
+                    GlobalAlloc::Function { instance, .. } => (
+                        self.get_fn_addr(instance),
                         self.data_layout().instruction_address_space,
                     ),
+                    GlobalAlloc::VTable(ty, dyn_ty) => {
+                        let alloc = self
+                            .tcx
+                            .global_alloc(self.tcx.vtable_allocation((
+                                ty,
+                                dyn_ty.principal().map(|principal| {
+                                    self.tcx.instantiate_bound_regions_with_erased(principal)
+                                }),
+                            )))
+                            .unwrap_memory();
+                        let init = const_alloc_to_llvm(self, alloc, /*static*/ false);
+                        let value = self.static_addr_of(init, alloc.inner().align, None);
+                        (value, AddressSpace::DATA)
+                    }
                     GlobalAlloc::Static(def_id) => {
                         assert!(self.tcx.is_static(def_id));
                         assert!(!self.tcx.is_thread_local_static(def_id));
                         let val = self.get_static(def_id);
-                        let addrspace = unsafe {
-                            llvm::LLVMGetPointerAddressSpace(llvm::LLVMRustGetValueType(val))
-                        };
-                        (self.get_static(def_id), AddressSpace(addrspace))
+                        let addrspace =
+                            unsafe { llvm::LLVMGetPointerAddressSpace(self.val_ty(val)) };
+                        (val, AddressSpace(addrspace))
                     }
                 };
                 let llval = unsafe {
-                    llvm::LLVMConstInBoundsGEP(
-                        self.const_bitcast(base_addr, self.type_i8p_ext(base_addr_space)),
+                    llvm::LLVMConstInBoundsGEP2(
+                        self.type_i8(),
+                        // Cast to the required address space if necessary
+                        self.const_bitcast(base_addr, self.type_ptr_ext(base_addr_space)),
                         &self.const_usize(offset.bytes()),
                         1,
                     )
                 };
 
-                if layout.value != Pointer {
+                if !matches!(layout.primitive(), Pointer(_)) {
                     unsafe { llvm::LLVMConstPtrToInt(llval, llty) }
                 } else {
                     if base_addr_space != AddressSpace::DATA {
@@ -185,35 +247,44 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         val
     }
 
-    fn from_const_alloc(
-        &self,
-        layout: TyAndLayout<'tcx>,
-        alloc: &Allocation,
-        offset: Size,
-    ) -> PlaceRef<'tcx, &'ll Value> {
-        assert_eq!(alloc.align, layout.align.abi);
-        let llty = self.type_ptr_to(layout.llvm_type(self));
-        let llval = if layout.size == Size::ZERO {
-            let llval = self.const_usize(alloc.align.bytes());
-            unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
-        } else {
-            let init = const_alloc_to_llvm(self, alloc);
-            let base_addr = self.static_addr_of(init, alloc.align, None);
-
-            let llval = unsafe {
-                llvm::LLVMConstInBoundsGEP(
-                    self.const_bitcast(base_addr, self.type_i8p()),
-                    &self.const_usize(offset.bytes()),
-                    1,
-                )
-            };
-            self.const_bitcast(llval, llty)
-        };
-        PlaceRef::new_sized(llval, layout)
+    fn is_undef(&self, v: Self::Value) -> bool {
+        unsafe { llvm::LLVMIsUndef(v) == True }
+    }
+
+    fn const_poison(&self, t: Self::Type) -> Self::Value {
+        // FIXME: Use LLVMGetPoision when possible.
+        self.const_undef(t)
     }
 
-    fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value {
-        unsafe { llvm::LLVMConstPointerCast(val, ty) }
+    fn const_i8(&self, i: i8) -> Self::Value {
+        self.const_int(self.type_i8(), i as i64)
+    }
+
+    fn const_i16(&self, i: i16) -> Self::Value {
+        self.const_int(self.type_i16(), i as i64)
+    }
+
+    fn const_u128(&self, i: u128) -> Self::Value {
+        trace!("const_u128 i = {i:?}");
+        trace!("{}", std::backtrace::Backtrace::force_capture());
+        self.const_uint_big(self.type_i128(), i)
+    }
+
+    fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value {
+        let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow");
+        unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) }
+    }
+
+    fn const_ptr_byte_offset(&self, mut base_addr: Self::Value, offset: abi::Size) -> Self::Value {
+        base_addr = self.const_ptrcast(base_addr, self.type_i8p());
+        unsafe {
+            llvm::LLVMConstInBoundsGEP2(
+                self.type_i8(),
+                base_addr,
+                &self.const_usize(offset.bytes()),
+                1,
+            )
+        }
     }
 }
 
diff --git a/crates/rustc_codegen_nvvm/src/consts.rs b/crates/rustc_codegen_nvvm/src/consts.rs
index 82378afd..43941810 100644
--- a/crates/rustc_codegen_nvvm/src/consts.rs
+++ b/crates/rustc_codegen_nvvm/src/consts.rs
@@ -1,26 +1,24 @@
 use std::ops::Range;
 
 use crate::debug_info;
-use crate::llvm::{self, Bool, True, Type, Value};
+use crate::llvm::{self, True, Type, Value};
 use libc::{c_char, c_uint};
-use rustc_codegen_ssa::traits::{
-    BaseTypeMethods, ConstMethods, DerivedTypeMethods, MiscMethods, StaticMethods,
-};
+use rustc_abi::{AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange};
+use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::DefId;
 use rustc_middle::mir::interpret::{
-    read_target_uint, Allocation, ErrorHandled, GlobalAlloc, Pointer,
+    Allocation, ConstAllocation, ErrorHandled, Pointer, read_target_uint,
 };
+use rustc_middle::ty::layout::HasTypingEnv;
+use rustc_middle::ty::{self, Instance, Ty};
 use rustc_middle::{
     bug,
     middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs},
-    mir::interpret::{InitChunk, Scalar as InterpScalar},
-    mir::mono::{Linkage, MonoItem},
+    mir::{
+        interpret::{InitChunk, Scalar as InterpScalar},
+        mono::{Linkage, MonoItem},
+    },
     span_bug,
-    ty::{self, layout::LayoutOf, Instance, Ty},
-};
-use rustc_span::Symbol;
-use rustc_target::abi::{
-    AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange,
 };
 use tracing::trace;
 
@@ -41,40 +39,26 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value {
         bytes_in_context(self.llcx, bytes)
     }
-
-    pub(crate) fn const_cstr(&self, s: Symbol, null_terminated: bool) -> &'ll Value {
-        trace!("Const cstr for symbol `{:?}`", s);
-        unsafe {
-            if let Some(&llval) = self.const_cstr_cache.borrow().get(&s) {
-                return llval;
-            }
-
-            let s_str = s.as_str();
-            let sc = llvm::LLVMConstStringInContext(
-                self.llcx,
-                s_str.as_ptr() as *const c_char,
-                s_str.len() as c_uint,
-                !null_terminated as Bool,
-            );
-            let sym = self.generate_local_symbol_name("str");
-            let g = self
-                .define_global(&sym[..], self.val_ty(sc), AddressSpace::DATA)
-                .unwrap_or_else(|| {
-                    bug!("symbol `{}` is already defined", sym);
-                });
-            llvm::LLVMSetInitializer(g, sc);
-            llvm::LLVMSetGlobalConstant(g, True);
-            llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage);
-
-            self.const_cstr_cache.borrow_mut().insert(s, g);
-            g
-        }
-    }
 }
 
-pub(crate) fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll Value {
+pub(crate) fn const_alloc_to_llvm<'ll>(
+    cx: &CodegenCx<'ll, '_>,
+    alloc: ConstAllocation<'_>,
+    is_static: bool,
+) -> &'ll Value {
     trace!("Const alloc to llvm");
-    let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
+    let alloc = alloc.inner();
+    // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or
+    // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be
+    // producing empty LLVM allocations as they're just adding noise to binaries and forcing less
+    // optimal codegen.
+    //
+    // Statics have a guaranteed meaningful address so it's less clear that we want to do
+    // something like this; it's also harder.
+    if !is_static {
+        assert!(alloc.len() != 0);
+    }
+    let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1);
     let dl = cx.data_layout();
     let pointer_size = dl.pointer_size.bytes() as usize;
 
@@ -86,9 +70,7 @@ pub(crate) fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocati
         alloc: &'a Allocation,
         range: Range<usize>,
     ) {
-        let mut chunks = alloc
-            .init_mask()
-            .range_as_init_chunks(Size::from_bytes(range.start), Size::from_bytes(range.end));
+        let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into());
 
         let chunk_to_llval = move |chunk| match chunk {
             InitChunk::Init(range) => {
@@ -102,39 +84,25 @@ pub(crate) fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocati
             }
         };
 
-        // Generating partially-uninit consts inhibits optimizations, so it is disabled by default.
-        // See https://github.com/rust-lang/rust/issues/84565.
-        let allow_partially_uninit = match cx
-            .sess()
-            .opts
-            .debugging_opts
-            .partially_uninit_const_threshold
-        {
-            Some(max) => range.len() <= max,
-            None => false,
-        };
+        // Generating partially-uninit consts is limited to small numbers of chunks,
+        // to avoid the cost of generating large complex const expressions.
+        // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element,
+        // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`.
+        let max = cx.sess().opts.unstable_opts.uninit_const_chunk_threshold;
+        let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max;
 
-        if allow_partially_uninit {
+        if allow_uninit_chunks {
             llvals.extend(chunks.map(chunk_to_llval));
         } else {
-            let llval = match (chunks.next(), chunks.next()) {
-                (Some(chunk), None) => {
-                    // exactly one chunk, either fully init or fully uninit
-                    chunk_to_llval(chunk)
-                }
-                _ => {
-                    // partially uninit, codegen as if it was initialized
-                    // (using some arbitrary value for uninit bytes)
-                    let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range);
-                    cx.const_bytes(bytes)
-                }
-            };
-            llvals.push(llval);
+            // If this allocation contains any uninit bytes, codegen as if it was initialized
+            // (using some arbitrary value for uninit bytes).
+            let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range);
+            llvals.push(cx.const_bytes(bytes));
         }
     }
 
     let mut next_offset = 0;
-    for &(offset, alloc_id) in alloc.relocations().iter() {
+    for &(offset, prov) in alloc.provenance().ptrs().iter() {
         let offset = offset.bytes();
         assert_eq!(offset as usize as u64, offset);
         let offset = offset as usize;
@@ -154,21 +122,15 @@ pub(crate) fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocati
         .expect("const_alloc_to_llvm: could not read relocation pointer")
             as u64;
 
-        let address_space = match cx.tcx.global_alloc(alloc_id) {
-            GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space,
-            GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) => AddressSpace::DATA,
-        };
+        let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx);
 
         llvals.push(cx.scalar_to_backend(
-            InterpScalar::from_pointer(
-                Pointer::new(alloc_id, Size::from_bytes(ptr_offset)),
-                &cx.tcx,
-            ),
-            Scalar {
-                value: Primitive::Pointer,
+            InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx),
+            Scalar::Initialized {
+                value: Primitive::Pointer(address_space),
                 valid_range: WrappingRange { start: 0, end: !0 },
             },
-            cx.type_i8p_ext(address_space),
+            cx.type_ptr_ext(address_space),
         ));
         next_offset = offset + pointer_size;
     }
@@ -186,9 +148,9 @@ pub(crate) fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocati
 pub(crate) fn codegen_static_initializer<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     def_id: DefId,
-) -> Result<(&'ll Value, &'tcx Allocation), ErrorHandled> {
+) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> {
     let alloc = cx.tcx.eval_static_initializer(def_id)?;
-    Ok((const_alloc_to_llvm(cx, alloc), alloc))
+    Ok((const_alloc_to_llvm(cx, alloc, /*static*/ true), alloc))
 }
 
 pub(crate) fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage {
@@ -199,9 +161,7 @@ pub(crate) fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage {
         Linkage::LinkOnceODR => llvm::Linkage::LinkOnceODRLinkage,
         Linkage::WeakAny => llvm::Linkage::WeakAnyLinkage,
         Linkage::WeakODR => llvm::Linkage::WeakODRLinkage,
-        Linkage::Appending => llvm::Linkage::AppendingLinkage,
         Linkage::Internal => llvm::Linkage::InternalLinkage,
-        Linkage::Private => llvm::Linkage::PrivateLinkage,
         Linkage::ExternalWeak => llvm::Linkage::ExternalWeakLinkage,
         Linkage::Common => llvm::Linkage::CommonLinkage,
     }
@@ -221,9 +181,12 @@ fn check_and_apply_linkage<'ll, 'tcx>(
         // https://docs.nvidia.com/cuda/nvvm-ir-spec/index.html#linkage-types-nvvm
         use Linkage::*;
         match linkage {
-            External | Private | Internal | Common | AvailableExternally | LinkOnceAny
-            | LinkOnceODR | WeakAny | WeakODR => {}
-            _ => cx.fatal(&format!("Unsupported linkage kind: {:?}", linkage)),
+            External | Internal | Common | AvailableExternally | LinkOnceAny | LinkOnceODR
+            | WeakAny | WeakODR => {}
+            _ => cx
+                .sess()
+                .dcx()
+                .fatal(format!("Unsupported linkage kind: {:?}", linkage)),
         }
 
         // If this is a static with a linkage specified, then we need to handle
@@ -231,10 +194,10 @@ fn check_and_apply_linkage<'ll, 'tcx>(
         // extern "C" fn() from being non-null, so we can't just declare a
         // static and call it a day. Some linkages (like weak) will make it such
         // that the static actually has a null value.
-        let llty2 = if let ty::RawPtr(ref mt) = ty.kind() {
-            cx.layout_of(mt.ty).llvm_type(cx)
+        let llty2 = if let ty::RawPtr(ty2, _) = ty.kind() {
+            cx.layout_of(ty2.clone()).llvm_type(cx)
         } else {
-            cx.sess().span_fatal(
+            cx.sess().dcx().span_fatal(
                 cx.tcx.def_span(span_def_id),
                 "must have type `*const T` or `*mut T` due to `#[linkage]` attribute",
             )
@@ -255,9 +218,9 @@ fn check_and_apply_linkage<'ll, 'tcx>(
             let g2 = cx
                 .define_global(&real_name, llty, addrspace)
                 .unwrap_or_else(|| {
-                    cx.sess().span_fatal(
+                    cx.sess().dcx().span_fatal(
                         cx.tcx.def_span(span_def_id),
-                        &format!("symbol `{}` is already defined", &sym),
+                        format!("symbol `{}` is already defined", &sym),
                     )
                 });
             llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage);
@@ -269,12 +232,17 @@ fn check_and_apply_linkage<'ll, 'tcx>(
     }
 }
 
-impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
+impl<'ll> CodegenCx<'ll, '_> {
     pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value {
         trace!("Const bitcast: `{:?}` to `{:?}`", val, ty);
         unsafe { llvm::LLVMConstBitCast(val, ty) }
     }
 
+    pub(crate) fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value {
+        trace!("Const ptrcast: `{:?}` to `{:?}`", val, ty);
+        unsafe { llvm::LLVMConstPointerCast(val, ty) }
+    }
+
     pub(crate) fn static_addr_of_mut(
         &self,
         cv: &'ll Value,
@@ -312,7 +280,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
             def_id
         );
 
-        let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
+        let ty = instance.ty(self.tcx, self.typing_env());
         let sym = self.tcx.symbol_name(instance).name;
         let fn_attrs = self.tcx.codegen_fn_attrs(def_id);
 
@@ -347,7 +315,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     }
 }
 
-impl<'ll, 'tcx> StaticMethods for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> StaticCodegenMethods for CodegenCx<'ll, 'tcx> {
     fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value {
         if let Some(&gv) = self.const_globals.borrow().get(&cv) {
             unsafe {
@@ -368,8 +336,17 @@ impl<'ll, 'tcx> StaticMethods for CodegenCx<'ll, 'tcx> {
         gv
     }
 
-    fn codegen_static(&self, def_id: DefId, is_mutable: bool) {
+    fn codegen_static(&self, def_id: DefId) {
         unsafe {
+            assert!(
+                llvm::LLVMGetInitializer(
+                    self.instances
+                        .borrow()
+                        .get(&Instance::mono(self.tcx, def_id))
+                        .unwrap()
+                )
+                .is_none()
+            );
             let attrs = self.tcx.codegen_fn_attrs(def_id);
 
             let (v, _) = match codegen_static_initializer(self, def_id) {
@@ -389,16 +366,14 @@ impl<'ll, 'tcx> StaticMethods for CodegenCx<'ll, 'tcx> {
             };
 
             let instance = Instance::mono(self.tcx, def_id);
-            let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
+            let ty = instance.ty(self.tcx, self.typing_env());
             let llty = self.layout_of(ty).llvm_type(self);
             let g = if val_llty == llty {
                 g
             } else {
                 trace!(
                     "Making new RAUW global: from ty `{:?}` to `{:?}`, initializer: `{:?}`",
-                    llty,
-                    val_llty,
-                    v
+                    llty, val_llty, v
                 );
                 // If we created the global with the wrong type,
                 // correct the type.
@@ -431,17 +406,17 @@ impl<'ll, 'tcx> StaticMethods for CodegenCx<'ll, 'tcx> {
             llvm::LLVMSetAlignment(g, self.align_of(ty).bytes() as c_uint);
             llvm::LLVMSetInitializer(g, v);
 
-            debug_info::create_global_var_metadata(self, def_id, g);
+            debug_info::build_global_var_di_node(self, def_id, g);
 
             // As an optimization, all shared statics which do not have interior
             // mutability are placed into read-only memory.
-            if !is_mutable && self.type_is_freeze(ty) {
+            if self.type_is_freeze(ty) {
                 // TODO(RDambrosio016): is this the same as putting this in
                 // the __constant__ addrspace for nvvm? should we set this addrspace explicitly?
                 // llvm::LLVMSetGlobalConstant(g, llvm::True);
             }
 
-            debug_info::create_global_var_metadata(self, def_id, g);
+            debug_info::build_global_var_di_node(self, def_id, g);
 
             if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) {
                 self.unsupported("thread locals");
diff --git a/crates/rustc_codegen_nvvm/src/context.rs b/crates/rustc_codegen_nvvm/src/context.rs
index 49ea6507..795ee946 100644
--- a/crates/rustc_codegen_nvvm/src/context.rs
+++ b/crates/rustc_codegen_nvvm/src/context.rs
@@ -1,44 +1,47 @@
+use std::cell::{Cell, RefCell};
+use std::ffi::CStr;
+use std::path::PathBuf;
+use std::ptr::null;
+use std::str::FromStr;
+
 use crate::abi::FnAbiLlvmExt;
 use crate::attributes::{self, NvvmAttributes, Symbols};
-use crate::debug_info::{self, compile_unit_metadata, CrateDebugContext};
+use crate::debug_info::{self, CodegenUnitDebugContext};
 use crate::llvm::{self, BasicBlock, Type, Value};
-use crate::{target, LlvmMod};
+use crate::{LlvmMod, target};
 use nvvm::NvvmOption;
-use rustc_codegen_ssa::traits::{BackendTypes, BaseTypeMethods, CoverageInfoMethods, MiscMethods};
-use rustc_codegen_ssa::traits::{ConstMethods, DerivedTypeMethods};
-use rustc_data_structures::base_n;
+use rustc_abi::AddressSpace;
+use rustc_abi::{HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx};
+use rustc_codegen_ssa::errors as ssa_errors;
+use rustc_codegen_ssa::traits::{
+    BackendTypes, BaseTypeCodegenMethods, CoverageInfoBuilderMethods, DerivedTypeCodegenMethods,
+    MiscCodegenMethods,
+};
+use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN};
+use rustc_errors::DiagMessage;
 use rustc_hash::FxHashMap;
 use rustc_middle::dep_graph::DepContext;
 use rustc_middle::ty::layout::{
-    FnAbiError, FnAbiOf, FnAbiRequest, HasParamEnv, HasTyCtxt, LayoutError, TyAndLayout,
+    FnAbiError, FnAbiOf, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError,
 };
 use rustc_middle::ty::layout::{FnAbiOfHelpers, LayoutOfHelpers};
-use rustc_middle::ty::{Ty, TypeFoldable};
+use rustc_middle::ty::{Ty, TypeVisitableExt};
 use rustc_middle::{bug, span_bug, ty};
 use rustc_middle::{
     mir::mono::CodegenUnit,
-    ty::{Instance, PolyExistentialTraitRef, TyCtxt},
+    ty::{Instance, TyCtxt},
 };
-use rustc_session::config::DebugInfo;
 use rustc_session::Session;
+use rustc_session::config::DebugInfo;
+use rustc_span::source_map::Spanned;
 use rustc_span::{Span, Symbol};
-use rustc_target::abi::call::FnAbi;
-use rustc_target::abi::{
-    AddressSpace, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx,
-};
+use rustc_target::callconv::FnAbi;
+
 use rustc_target::spec::{HasTargetSpec, Target};
-use std::cell::{Cell, RefCell};
-use std::ffi::CStr;
-use std::hash::BuildHasherDefault;
-use std::os::raw::c_char;
-use std::path::PathBuf;
-use std::ptr::null;
-use std::str::FromStr;
 use tracing::{debug, trace};
 
 pub(crate) struct CodegenCx<'ll, 'tcx> {
     pub tcx: TyCtxt<'tcx>,
-    pub check_overflow: bool,
 
     pub llmod: &'ll llvm::Module,
     pub llcx: &'ll llvm::Context,
@@ -47,9 +50,9 @@ pub(crate) struct CodegenCx<'ll, 'tcx> {
     /// Map of MIR functions to LLVM function values
     pub instances: RefCell<FxHashMap<Instance<'tcx>, &'ll Value>>,
     /// A cache of the generated vtables for trait objects
-    pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>), &'ll Value>>,
+    pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), &'ll Value>>,
     /// A cache of constant strings and their values
-    pub const_cstr_cache: RefCell<FxHashMap<Symbol, &'ll Value>>,
+    pub const_cstr_cache: RefCell<FxHashMap<String, &'ll Value>>,
     /// A map of functions which have parameters at specific indices replaced with an int-remapped type.
     /// such as i128 --> <2 x i64>
     #[allow(clippy::type_complexity)]
@@ -78,10 +81,10 @@ pub(crate) struct CodegenCx<'ll, 'tcx> {
     pub pointee_infos: RefCell<FxHashMap<(Ty<'tcx>, Size), Option<PointeeInfo>>>,
     pub isize_ty: &'ll Type,
 
-    pub dbg_cx: Option<debug_info::CrateDebugContext<'ll, 'tcx>>,
+    pub dbg_cx: Option<debug_info::CodegenUnitDebugContext<'ll, 'tcx>>,
 
     /// A map of the intrinsics we actually declared for usage.
-    pub(crate) intrinsics: RefCell<FxHashMap<String, &'ll Value>>,
+    pub(crate) intrinsics: RefCell<FxHashMap<String, (&'ll Type, &'ll Value)>>,
     /// A map of the intrinsics available but not yet declared.
     pub(crate) intrinsics_map: RefCell<FxHashMap<&'static str, (Vec<&'ll Type>, &'ll Type)>>,
 
@@ -106,7 +109,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         llvm_module: &'ll LlvmMod,
     ) -> Self {
         debug!("Creating new CodegenCx");
-        let check_overflow = tcx.sess.overflow_checks();
         let (llcx, llmod) = (&*llvm_module.llcx, unsafe {
             llvm_module.llmod.as_ref().unwrap()
         });
@@ -122,8 +124,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         };
 
         let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None {
-            let dctx = CrateDebugContext::new(llmod);
-            compile_unit_metadata(tcx, &codegen_unit.name().as_str(), &dctx);
+            let dctx = CodegenUnitDebugContext::new(llmod);
+            debug_info::build_compile_unit_di_node(tcx, &codegen_unit.name().as_str(), &dctx);
             Some(dctx)
         } else {
             None
@@ -131,7 +133,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
 
         let mut cx = CodegenCx {
             tcx,
-            check_overflow,
             llmod,
             llcx,
             codegen_unit,
@@ -151,10 +152,13 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
             intrinsics_map: RefCell::new(FxHashMap::with_capacity_and_hasher(
                 // ~319 libdevice intrinsics plus some headroom for llvm
                 350,
-                BuildHasherDefault::default(),
+                Default::default(),
             )),
             local_gen_sym_counter: Cell::new(0),
-            nvptx_data_layout: TargetDataLayout::parse(&target::target()).unwrap(),
+            nvptx_data_layout: TargetDataLayout::parse_from_llvm_datalayout_string(
+                &target::target().data_layout,
+            )
+            .unwrap_or_else(|err| tcx.sess.dcx().emit_fatal(err)),
             nvptx_target: target::target(),
             eh_personality,
             symbols: Symbols {
@@ -170,29 +174,29 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         cx
     }
 
-    pub(crate) fn fatal(&self, msg: &str) -> ! {
-        self.tcx.sess.fatal(msg)
+    pub(crate) fn fatal(&self, msg: impl Into<DiagMessage>) -> ! {
+        self.tcx.sess.dcx().fatal(msg)
     }
 
     // im lazy i know
     pub(crate) fn unsupported(&self, thing: &str) -> ! {
-        self.fatal(&format!("{} is unsupported", thing))
+        self.fatal(format!("{} is unsupported", thing))
     }
 
-    fn create_used_variable_impl(&self, name: *const c_char, values: &[&'ll Value]) {
-        let section = "llvm.metadata\0".as_ptr().cast();
+    pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) {
+        let section = c"llvm.metadata";
         let array = self.const_array(self.type_ptr_to(self.type_i8()), values);
 
         unsafe {
             trace!(
                 "Creating LLVM used variable with name `{}` and values:\n{:#?}",
-                CStr::from_ptr(name).to_str().unwrap(),
+                name.to_str().unwrap(),
                 values
             );
-            let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name);
+            let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr());
             llvm::LLVMSetInitializer(g, array);
             llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage);
-            llvm::LLVMSetSection(g, section);
+            llvm::LLVMSetSection(g, section.as_ptr());
         }
     }
 }
@@ -201,10 +205,10 @@ fn sanitize_global_ident(name: &str) -> String {
     name.replace(".", "$")
 }
 
-impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn vtables(
         &self,
-    ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>), &'ll Value>> {
+    ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), &'ll Value>> {
         &self.vtables
     }
 
@@ -224,51 +228,38 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         self.tcx.sess
     }
 
-    fn check_overflow(&self) -> bool {
-        self.check_overflow
-    }
-
     fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> {
         self.codegen_unit
     }
 
-    fn used_statics(&self) -> &RefCell<Vec<&'ll Value>> {
-        &self.used_statics
-    }
-
-    fn create_used_variable(&self) {
-        self.create_used_variable_impl("llvm.used\0".as_ptr().cast(), &*self.used_statics.borrow());
-    }
-
-    fn create_compiler_used_variable(&self) {
-        self.create_used_variable_impl(
-            "llvm.compiler.used\0".as_ptr().cast(),
-            &*self.compiler_used_statics.borrow(),
-        );
-    }
-
-    fn declare_c_main(&self, _fn_type: Self::Type) -> Option<Self::Function> {
+    fn declare_c_main(
+        &self,
+        _fn_type: <CodegenCx<'ll, 'tcx> as rustc_codegen_ssa::traits::BackendTypes>::Type,
+    ) -> Option<<CodegenCx<'ll, 'tcx> as rustc_codegen_ssa::traits::BackendTypes>::Function> {
         // no point for gpu kernels
         None
     }
 
-    fn apply_target_cpu_attr(&self, _llfn: Self::Function) {
+    fn apply_target_cpu_attr(
+        &self,
+        _llfn: <CodegenCx<'ll, 'tcx> as rustc_codegen_ssa::traits::BackendTypes>::Function,
+    ) {
         // no point if we are running on the gpu ;)
     }
 
-    fn compiler_used_statics(&self) -> &RefCell<Vec<Self::Value>> {
-        &self.compiler_used_statics
+    fn set_frame_pointer_type(
+        &self,
+        _llfn: <CodegenCx<'ll, 'tcx> as rustc_codegen_ssa::traits::BackendTypes>::Function,
+    ) {
     }
-
-    fn set_frame_pointer_type(&self, _llfn: Self::Function) {}
 }
 
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     /// Computes the address space for a static.
     pub fn static_addrspace(&self, instance: Instance<'tcx>) -> AddressSpace {
-        let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
+        let ty = instance.ty(self.tcx, self.typing_env());
         let is_mutable = self.tcx().is_mutable_static(instance.def_id());
-        let attrs = self.tcx.get_attrs(instance.def_id());
+        let attrs = self.tcx.get_attrs_unchecked(instance.def_id()); // TODO: replace with get_attrs
         let nvvm_attrs = NvvmAttributes::parse(self, attrs);
 
         if let Some(addr) = nvvm_attrs.addrspace {
@@ -371,15 +362,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     pub fn get_defined_value(&self, name: &str) -> Option<&'ll Value> {
         self.get_declared_value(name).and_then(|val| {
             let declaration = unsafe { llvm::LLVMIsDeclaration(val) != 0 };
-            if !declaration {
-                Some(val)
-            } else {
-                None
-            }
+            if !declaration { Some(val) } else { None }
         })
     }
 
-    pub(crate) fn get_intrinsic(&self, key: &str) -> &'ll Value {
+    pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) {
         trace!("Retrieving intrinsic with name `{}`", key);
         if let Some(v) = self.intrinsics.borrow().get(key).cloned() {
             return v;
@@ -391,10 +378,10 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
 
     pub(crate) fn insert_intrinsic(
         &self,
-        name: String,
+        name: &str,
         args: Option<&[&'ll Type]>,
         ret: &'ll Type,
-    ) -> &'ll Value {
+    ) -> (&'ll Type, &'ll Value) {
         let fn_ty = if let Some(args) = args {
             self.type_func(args, ret)
         } else {
@@ -402,8 +389,10 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         };
         let f = self.declare_fn(&name, fn_ty, None);
         llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No);
-        self.intrinsics.borrow_mut().insert(name, f);
-        f
+        self.intrinsics
+            .borrow_mut()
+            .insert(name.to_owned(), (fn_ty, f));
+        (fn_ty, f)
     }
 
     pub fn generate_local_symbol_name(&self, prefix: &str) -> String {
@@ -414,7 +403,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         let mut name = String::with_capacity(prefix.len() + 6);
         name.push_str(prefix);
         name.push('.');
-        base_n::push_str(idx as u128, base_n::ALPHANUMERIC_ONLY, &mut name);
+        name.push_str(&(idx as u64).to_base(ALPHANUMERIC_ONLY));
         name
     }
 
@@ -422,39 +411,45 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     pub fn get_fn(&self, instance: Instance<'tcx>) -> &'ll Value {
         let tcx = self.tcx;
 
-        assert!(!instance.substs.needs_infer());
-        assert!(!instance.substs.has_escaping_bound_vars());
-        let sym = tcx.symbol_name(instance).name;
+        assert!(!instance.args.has_infer());
+        assert!(!instance.args.has_escaping_bound_vars());
 
         if let Some(&llfn) = self.instances.borrow().get(&instance) {
             return llfn;
         }
 
-        let abi = self.fn_abi_of_instance(instance, ty::List::empty());
+        let sym = tcx.symbol_name(instance).name;
+        debug!(
+            "get_fn({:?}: {:?}) => {}",
+            instance,
+            instance.ty(self.tcx(), self.typing_env()),
+            sym
+        );
+
+        let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
 
         let llfn = if let Some(llfn) = self.get_declared_value(sym) {
             trace!("Returning existing llfn `{:?}`", llfn);
-            let llptrty = abi.ptr_to_llvm_type(self);
+            let llptrty = fn_abi.ptr_to_llvm_type(self);
 
             if self.val_ty(llfn) != llptrty {
                 trace!(
                     "ptrcasting llfn to different llptrty: `{:?}` --> `{:?}`",
-                    llfn,
-                    llptrty
+                    llfn, llptrty
                 );
                 self.const_ptrcast(llfn, llptrty)
             } else {
                 llfn
             }
         } else {
-            let llfn = self.declare_fn(sym, abi.llvm_type(self), Some(abi));
+            let llfn = self.declare_fn(sym, fn_abi.llvm_type(self), Some(fn_abi));
             attributes::from_fn_attrs(self, llfn, instance);
             let def_id = instance.def_id();
 
             unsafe {
                 llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage);
 
-                let is_generic = instance.substs.non_erasable_generics().next().is_some();
+                let is_generic = instance.args.non_erasable_generics().next().is_some();
 
                 // nvvm ignores visibility styles, but we still make them just in case it will do something
                 // with them in the future or we want to use that metadata
@@ -559,13 +554,15 @@ impl<'ll, 'tcx> BackendTypes for CodegenCx<'ll, 'tcx> {
 
     type BasicBlock = &'ll BasicBlock;
     type Type = &'ll Type;
-    // not applicable to nvvm, unwinding/exception handling on the gpu
-    // doesnt exist.
+    // not applicable to nvvm, unwinding/exception handling
+    // doesnt exist on the gpu
     type Funclet = ();
 
     type DIScope = &'ll llvm::DIScope;
     type DILocation = &'ll llvm::DILocation;
     type DIVariable = &'ll llvm::DIVariable;
+
+    type Metadata = &'ll llvm::Metadata;
 }
 
 impl<'ll, 'tcx> HasDataLayout for CodegenCx<'ll, 'tcx> {
@@ -586,28 +583,29 @@ impl<'ll, 'tcx> ty::layout::HasTyCtxt<'tcx> for CodegenCx<'ll, 'tcx> {
     }
 }
 
-impl<'ll, 'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'ll, 'tcx> {
-    type LayoutOfResult = TyAndLayout<'tcx>;
+impl<'tcx, 'll> HasTypingEnv<'tcx> for CodegenCx<'ll, 'tcx> {
+    fn typing_env<'a>(&'a self) -> ty::TypingEnv<'tcx> {
+        ty::TypingEnv::fully_monomorphized()
+    }
+}
 
+impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> {
     #[inline]
     fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
-        if let LayoutError::SizeOverflow(_) = err {
-            self.sess().span_fatal(span, &err.to_string())
+        if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err {
+            self.tcx.dcx().emit_fatal(Spanned {
+                span,
+                node: err.into_diagnostic(),
+            })
         } else {
-            span_bug!(span, "failed to get layout for `{}`: {}", ty, err)
+            self.tcx
+                .dcx()
+                .emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err })
         }
     }
 }
 
-impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> {
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        ty::ParamEnv::reveal_all()
-    }
-}
-
 impl<'ll, 'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'ll, 'tcx> {
-    type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>;
-
     #[inline]
     fn handle_fn_abi_err(
         &self,
@@ -615,17 +613,15 @@ impl<'ll, 'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'ll, 'tcx> {
         span: Span,
         fn_abi_request: FnAbiRequest<'tcx>,
     ) -> ! {
-        if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err {
-            self.tcx.sess.span_fatal(span, &err.to_string())
-        } else {
-            match fn_abi_request {
+        match err {
+            FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => {
+                self.tcx.dcx().emit_fatal(Spanned { span, node: err });
+            }
+            _ => match fn_abi_request {
                 FnAbiRequest::OfFnPtr { sig, extra_args } => {
                     span_bug!(
                         span,
-                        "`fn_abi_of_fn_ptr({}, {:?})` failed: {}",
-                        sig,
-                        extra_args,
-                        err
+                        "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}",
                     );
                 }
                 FnAbiRequest::OfInstance {
@@ -634,27 +630,24 @@ impl<'ll, 'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'ll, 'tcx> {
                 } => {
                     span_bug!(
                         span,
-                        "`fn_abi_of_instance({}, {:?})` failed: {}",
-                        instance,
-                        extra_args,
-                        err
+                        "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}",
                     );
                 }
-            }
+            },
         }
     }
 }
 
-impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
-    fn coverageinfo_finalize(&self) {
-        todo!()
-    }
-
-    fn define_unused_fn(&self, _def_id: rustc_hir::def_id::DefId) {
+impl<'ll, 'tcx> CoverageInfoBuilderMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+    fn init_coverage(&mut self, _instance: Instance<'tcx>) {
         todo!()
     }
 
-    fn get_pgo_func_name_var(&self, _instance: Instance<'tcx>) -> Self::Value {
+    fn add_coverage(
+        &mut self,
+        _instance: Instance<'tcx>,
+        _kind: &rustc_middle::mir::coverage::CoverageKind,
+    ) {
         todo!()
     }
 }
diff --git a/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs b/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs
index 5dae31f0..02d939d5 100644
--- a/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs
+++ b/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs
@@ -1,13 +1,14 @@
 use crate::context::CodegenCx;
+use crate::llvm::Type;
 use crate::llvm::Value;
-use rustc_codegen_ssa::traits::{BaseTypeMethods, DerivedTypeMethods};
+use rustc_codegen_ssa::traits::BaseTypeCodegenMethods;
 use rustc_session::config::DebugInfo;
 
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
-    pub(crate) fn declare_intrinsic(&self, key: &str) -> Option<&'ll Value> {
+    pub(crate) fn declare_intrinsic(&self, key: &str) -> Option<(&'ll Type, &'ll Value)> {
         let map = self.intrinsics_map.borrow();
         let (args, ret) = map.get(key)?;
-        Some(self.insert_intrinsic(key.to_string(), Some(args), ret))
+        Some(self.insert_intrinsic(key, Some(args), ret))
     }
 
     #[rustfmt::skip] // stop rustfmt from making this 2k lines
diff --git a/crates/rustc_codegen_nvvm/src/debug_info/create_scope_map.rs b/crates/rustc_codegen_nvvm/src/debug_info/create_scope_map.rs
index 0477f324..28b1b2b0 100644
--- a/crates/rustc_codegen_nvvm/src/debug_info/create_scope_map.rs
+++ b/crates/rustc_codegen_nvvm/src/debug_info/create_scope_map.rs
@@ -1,6 +1,11 @@
+use std::collections::hash_map::Entry;
+
 use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext};
 use rustc_codegen_ssa::traits::*;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::ty::layout::FnAbiOf;
+use rustc_middle::ty::layout::HasTypingEnv;
+use rustc_span::BytePos;
 
 use crate::context::CodegenCx;
 use crate::llvm;
@@ -9,8 +14,8 @@ use rustc_middle::mir::{Body, SourceScope};
 use rustc_middle::ty::{self, Instance};
 use rustc_session::config::DebugInfo;
 
-use rustc_index::bit_set::BitSet;
-use rustc_index::vec::Idx;
+use rustc_index::Idx;
+use rustc_index::bit_set::*;
 
 use super::metadata::file_metadata;
 use super::util::DIB;
@@ -20,19 +25,27 @@ pub(crate) fn compute_mir_scopes<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     instance: Instance<'tcx>,
     mir: &Body<'tcx>,
-    fn_dbg_scope: &'ll DIScope,
     debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>,
 ) {
     // Find all the scopes with variables defined in them.
-    let mut has_variables = BitSet::new_empty(mir.source_scopes.len());
-
-    // Only consider variables when they're going to be emitted.
-    if cx.sess().opts.debuginfo == DebugInfo::Full {
+    let variables = if cx.sess().opts.debuginfo == DebugInfo::Full {
+        // Only consider variables when they're going to be emitted.
+        let mut vars = DenseBitSet::new_empty(mir.source_scopes.len());
+        // FIXME(eddyb) take into account that arguments always have debuginfo,
+        // irrespective of their name (assuming full debuginfo is enabled).
+        // NOTE(eddyb) actually, on second thought, those are always in the
+        // function scope, which always exists.
         for var_debug_info in &mir.var_debug_info {
-            has_variables.insert(var_debug_info.source_info.scope);
+            vars.insert(var_debug_info.source_info.scope);
         }
-    }
+        Some(vars)
+    } else {
+        // Nothing to emit, of course.
+        None
+    };
 
+    let mut instantiated = DenseBitSet::new_empty(mir.source_scopes.len());
+    let mut discriminators = FxHashMap::default();
     // Instantiate all scopes.
     for idx in 0..mir.source_scopes.len() {
         let scope = SourceScope::new(idx);
@@ -40,9 +53,10 @@ pub(crate) fn compute_mir_scopes<'ll, 'tcx>(
             cx,
             instance,
             mir,
-            fn_dbg_scope,
-            &has_variables,
+            &variables,
             debug_context,
+            &mut instantiated,
+            &mut discriminators,
             scope,
         );
     }
@@ -52,12 +66,13 @@ fn make_mir_scope<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     instance: Instance<'tcx>,
     mir: &Body<'tcx>,
-    fn_dbg_scope: &'ll DIScope,
-    has_variables: &BitSet<SourceScope>,
+    variables: &Option<DenseBitSet<SourceScope>>,
     debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>,
+    instantiated: &mut DenseBitSet<SourceScope>,
+    discriminators: &mut FxHashMap<BytePos, u32>,
     scope: SourceScope,
 ) {
-    if debug_context.scopes[scope].dbg_scope.is_some() {
+    if instantiated.contains(scope) {
         return;
     }
 
@@ -67,9 +82,10 @@ fn make_mir_scope<'ll, 'tcx>(
             cx,
             instance,
             mir,
-            fn_dbg_scope,
-            has_variables,
+            variables,
             debug_context,
+            instantiated,
+            discriminators,
             parent,
         );
         debug_context.scopes[parent]
@@ -77,18 +93,21 @@ fn make_mir_scope<'ll, 'tcx>(
         // The root is the function itself.
         let loc = cx.lookup_debug_loc(mir.span.lo());
         debug_context.scopes[scope] = DebugScope {
-            dbg_scope: Some(fn_dbg_scope),
-            inlined_at: None,
             file_start_pos: loc.file.start_pos,
-            file_end_pos: loc.file.end_pos,
+            file_end_pos: loc.file.end_position(),
+            ..debug_context.scopes[scope]
         };
         return;
     };
 
-    if !has_variables.contains(scope) && scope_data.inlined.is_none() {
+    if let Some(vars) = variables
+        && !vars.contains(scope)
+        && scope_data.inlined.is_none()
+    {
         // Do not create a DIScope if there are no variables defined in this
         // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat.
         debug_context.scopes[scope] = parent_scope;
+        instantiated.insert(scope);
         return;
     }
 
@@ -97,10 +116,12 @@ fn make_mir_scope<'ll, 'tcx>(
 
     let dbg_scope = match scope_data.inlined {
         Some((callee, _)) => {
-            let callee = cx.tcx.subst_and_normalize_erasing_regions(
-                instance.substs,
-                ty::ParamEnv::reveal_all(),
-                callee,
+            // FIXME(eddyb) this would be `self.monomorphize(&callee)`
+            // if this is moved to `rustc_codegen_ssa::mir::debuginfo`.
+            let callee = cx.tcx.instantiate_and_normalize_erasing_regions(
+                instance.args,
+                cx.typing_env(),
+                ty::EarlyBinder::bind(callee),
             );
             let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty());
             cx.dbg_scope_fn(callee, callee_fn_abi, None)
@@ -108,7 +129,7 @@ fn make_mir_scope<'ll, 'tcx>(
         None => unsafe {
             llvm::LLVMRustDIBuilderCreateLexicalBlock(
                 DIB(cx),
-                parent_scope.dbg_scope.unwrap(),
+                parent_scope.dbg_scope,
                 file_metadata,
                 loc.line,
                 loc.col,
@@ -118,13 +139,43 @@ fn make_mir_scope<'ll, 'tcx>(
 
     let inlined_at = scope_data.inlined.map(|(_, callsite_span)| {
         let callsite_scope = parent_scope.adjust_dbg_scope_for_span(cx, callsite_span);
-        cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span)
+        let loc = cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span);
+
+        // NB: In order to produce proper debug info for variables (particularly
+        // arguments) in multiply-inline functions, LLVM expects to see a single
+        // DILocalVariable with multiple different DILocations in the IR. While
+        // the source information for each DILocation would be identical, their
+        // inlinedAt attributes will be unique to the particular callsite.
+        //
+        // We generate DILocations here based on the callsite's location in the
+        // source code. A single location in the source code usually can't
+        // produce multiple distinct calls so this mostly works, until
+        // proc-macros get involved. A proc-macro can generate multiple calls
+        // at the same span, which breaks the assumption that we're going to
+        // produce a unique DILocation for every scope we process here. We
+        // have to explicitly add discriminators if we see inlines into the
+        // same source code location.
+        //
+        // Note further that we can't key this hashtable on the span itself,
+        // because these spans could have distinct SyntaxContexts. We have
+        // to key on exactly what we're giving to LLVM.
+        match discriminators.entry(callsite_span.lo()) {
+            Entry::Occupied(mut o) => {
+                *o.get_mut() += 1;
+                unsafe { llvm::LLVMRustDILocationCloneWithBaseDiscriminator(loc, *o.get()) }
+                    .expect("Failed to encode discriminator in DILocation")
+            }
+            Entry::Vacant(v) => {
+                v.insert(0);
+                loc
+            }
+        }
     });
 
     debug_context.scopes[scope] = DebugScope {
-        dbg_scope: Some(dbg_scope),
+        dbg_scope: dbg_scope,
         inlined_at: inlined_at.or(parent_scope.inlined_at),
         file_start_pos: loc.file.start_pos,
-        file_end_pos: loc.file.end_pos,
+        file_end_pos: loc.file.end_position(),
     };
 }
diff --git a/crates/rustc_codegen_nvvm/src/debug_info/dwarf_const.rs b/crates/rustc_codegen_nvvm/src/debug_info/dwarf_const.rs
new file mode 100644
index 00000000..f4320950
--- /dev/null
+++ b/crates/rustc_codegen_nvvm/src/debug_info/dwarf_const.rs
@@ -0,0 +1,35 @@
+//! Definitions of various DWARF-related constants.
+
+use libc::c_uint;
+
+/// Helper macro to let us redeclare gimli's constants as our own constants
+/// with a different type, with less risk of copy-paste errors.
+macro_rules! declare_constant {
+    (
+        $name:ident : $type:ty
+    ) => {
+        #[allow(non_upper_case_globals)]
+        pub(crate) const $name: $type = ::gimli::constants::$name.0 as $type;
+
+        // Assert that as-cast probably hasn't changed the value.
+        const _: () = assert!($name as i128 == ::gimli::constants::$name.0 as i128);
+    };
+}
+
+// DWARF languages.
+declare_constant!(DW_LANG_Rust: c_uint);
+
+// DWARF attribute type encodings.
+declare_constant!(DW_ATE_boolean: c_uint);
+declare_constant!(DW_ATE_float: c_uint);
+declare_constant!(DW_ATE_signed: c_uint);
+declare_constant!(DW_ATE_unsigned: c_uint);
+declare_constant!(DW_ATE_UTF: c_uint);
+
+// DWARF expression operators.
+declare_constant!(DW_OP_deref: i64);
+declare_constant!(DW_OP_plus_uconst: i64);
+/// Defined by LLVM in `llvm/include/llvm/BinaryFormat/Dwarf.h`.
+/// Double-checked by a static assertion in `RustWrapper.cpp`.
+#[allow(non_upper_case_globals)]
+pub(crate) const DW_OP_LLVM_fragment: i64 = 0x1000;
diff --git a/crates/rustc_codegen_nvvm/src/debug_info/metadata.rs b/crates/rustc_codegen_nvvm/src/debug_info/metadata.rs
index 127946f7..fa7356fd 100644
--- a/crates/rustc_codegen_nvvm/src/debug_info/metadata.rs
+++ b/crates/rustc_codegen_nvvm/src/debug_info/metadata.rs
@@ -1,39 +1,36 @@
+use std::borrow::Cow;
+use std::ffi::CString;
+use std::fmt::{self, Write};
+use std::hash::{Hash, Hasher};
+use std::path::PathBuf;
+use std::{iter, ptr};
+
 use libc::{c_longlong, c_uint};
-use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name;
+use rustc_abi::{Align, Size};
+use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind;
 use rustc_codegen_ssa::traits::*;
-use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_hir::def::CtorKind;
+use rustc_hir::def::{CtorKind, DefKind};
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
-use rustc_index::vec::{Idx, IndexVec};
-use rustc_middle::mir::{self, GeneratorLayout};
-use rustc_middle::ty::layout::{self, IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout};
-use rustc_middle::ty::subst::GenericArgKind;
-use rustc_middle::ty::Instance;
-use rustc_middle::ty::{self, AdtKind, GeneratorSubsts, ParamEnv, Ty, TyCtxt};
-use rustc_middle::{bug, span_bug};
-use rustc_query_system::ich::NodeIdHashingMode;
+use rustc_middle::bug;
+use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf, TyAndLayout};
+use rustc_middle::ty::{self, AdtKind, CoroutineArgsExt, Instance, Ty, TyCtxt, Visibility};
 use rustc_session::config::{self, DebugInfo};
 use rustc_span::symbol::Symbol;
-use rustc_span::{self, SourceFile, SourceFileHash, Span};
-use rustc_span::{FileNameDisplayPreference, DUMMY_SP};
-use rustc_target::abi::{Abi, Align, HasDataLayout, Integer, TagEncoding};
-use rustc_target::abi::{
-    Primitive::{self, *},
-    Size, VariantIdx, Variants,
-};
-
+use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, hygiene};
+use smallvec::smallvec;
+use tracing::debug;
+
+pub(crate) use self::type_map::TypeMap;
+use self::type_map::{DINodeCreationResult, Stub, UniqueTypeId};
+use super::CodegenUnitDebugContext;
+use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name};
+use crate::common::AsCCharPtr;
 use crate::context::CodegenCx;
+use crate::debug_info::metadata::type_map::build_type_with_children;
 use crate::debug_info::util::*;
-use crate::llvm::debuginfo::*;
-use crate::llvm::{self, Value};
-use std::collections::hash_map::Entry;
-use std::ffi::CString;
-use std::hash::{Hash, Hasher};
-use std::path::PathBuf;
-use std::ptr;
-use std::{fmt, iter};
+use crate::debug_info::*;
+use crate::llvm::{DebugEmissionKind, Value};
+use crate::{abi, llvm};
 
 // most of this code is taken from rustc_codegen_llvm, but adapted
 // to use llvm 7 stuff. As well as removing some useless stuff to account for
@@ -59,347 +56,62 @@ impl fmt::Debug for llvm::Metadata {
     }
 }
 
-const DW_LANG_RUST: c_uint = 0x1c;
-#[allow(non_upper_case_globals)]
-const DW_ATE_boolean: c_uint = 0x02;
-#[allow(non_upper_case_globals)]
-const DW_ATE_float: c_uint = 0x04;
-#[allow(non_upper_case_globals)]
-const DW_ATE_signed: c_uint = 0x05;
-#[allow(non_upper_case_globals)]
-const DW_ATE_unsigned: c_uint = 0x07;
-#[allow(non_upper_case_globals)]
-const DW_ATE_unsigned_char: c_uint = 0x08;
-
-pub const UNKNOWN_LINE_NUMBER: c_uint = 0;
-pub const UNKNOWN_COLUMN_NUMBER: c_uint = 0;
-
-pub(crate) const NO_SCOPE_METADATA: Option<&DIScope> = None;
-
-mod unique_type_id {
-    use super::*;
-    use rustc_arena::DroplessArena;
-
-    #[derive(Copy, Hash, Eq, PartialEq, Clone)]
-    pub(super) struct UniqueTypeId(u32);
-
-    // The `&'static str`s in this type actually point into the arena.
-    //
-    // The `FxHashMap`+`Vec` pair could be replaced by `FxIndexSet`, but #75278
-    // found that to regress performance up to 2% in some cases. This might be
-    // revisited after further improvements to `indexmap`.
-    #[derive(Default)]
-    pub(super) struct TypeIdInterner {
-        arena: DroplessArena,
-        names: FxHashMap<&'static str, UniqueTypeId>,
-        strings: Vec<&'static str>,
-    }
-
-    impl TypeIdInterner {
-        #[inline]
-        pub(super) fn intern(&mut self, string: &str) -> UniqueTypeId {
-            if let Some(&name) = self.names.get(string) {
-                return name;
-            }
-
-            let name = UniqueTypeId(self.strings.len() as u32);
-
-            // `from_utf8_unchecked` is safe since we just allocated a `&str` which is known to be
-            // UTF-8.
-            let string: &str =
-                unsafe { std::str::from_utf8_unchecked(self.arena.alloc_slice(string.as_bytes())) };
-            // It is safe to extend the arena allocation to `'static` because we only access
-            // these while the arena is still alive.
-            let string: &'static str = unsafe { &*(string as *const str) };
-            self.strings.push(string);
-            self.names.insert(string, name);
-            name
-        }
-
-        // Get the symbol as a string. `Symbol::as_str()` should be used in
-        // preference to this function.
-        pub(super) fn get(&self, symbol: UniqueTypeId) -> &str {
-            self.strings[symbol.0 as usize]
-        }
-    }
-}
-use unique_type_id::*;
-
-/// The `TypeMap` is where the `CrateDebugContext` holds the type metadata nodes
-/// created so far. The metadata nodes are indexed by `UniqueTypeId`, and, for
-/// faster lookup, also by `Ty`. The `TypeMap` is responsible for creating
-/// `UniqueTypeId`s.
-#[derive(Default)]
-pub struct TypeMap<'ll, 'tcx> {
-    /// The `UniqueTypeId`s created so far.
-    unique_id_interner: TypeIdInterner,
-    /// A map from `UniqueTypeId` to debuginfo metadata for that type. This is a 1:1 mapping.
-    unique_id_to_metadata: FxHashMap<UniqueTypeId, &'ll DIType>,
-    /// A map from types to debuginfo metadata. This is an N:1 mapping.
-    type_to_metadata: FxHashMap<Ty<'tcx>, &'ll DIType>,
-    /// A map from types to `UniqueTypeId`. This is an N:1 mapping.
-    type_to_unique_id: FxHashMap<Ty<'tcx>, UniqueTypeId>,
-}
-
-impl<'ll, 'tcx> TypeMap<'ll, 'tcx> {
-    /// Adds a Ty to metadata mapping to the TypeMap. The method will fail if
-    /// the mapping already exists.
-    fn register_type_with_metadata(&mut self, type_: Ty<'tcx>, metadata: &'ll DIType) {
-        if self.type_to_metadata.insert(type_, metadata).is_some() {
-            bug!(
-                "type metadata for `Ty` '{}' is already in the `TypeMap`!",
-                type_
-            );
-        }
-    }
-
-    /// Removes a `Ty`-to-metadata mapping.
-    /// This is useful when computing the metadata for a potentially
-    /// recursive type (e.g., a function pointer of the form:
-    ///
-    ///     fn foo() -> impl Copy { foo }
-    ///
-    /// This kind of type cannot be properly represented
-    /// via LLVM debuginfo. As a workaround,
-    /// we register a temporary Ty to metadata mapping
-    /// for the function before we compute its actual metadata.
-    /// If the metadata computation ends up recursing back to the
-    /// original function, it will use the temporary mapping
-    /// for the inner self-reference, preventing us from
-    /// recursing forever.
-    ///
-    /// This function is used to remove the temporary metadata
-    /// mapping after we've computed the actual metadata.
-    fn remove_type(&mut self, type_: Ty<'tcx>) {
-        if self.type_to_metadata.remove(type_).is_none() {
-            bug!("type metadata `Ty` '{}' is not in the `TypeMap`!", type_);
-        }
-    }
-
-    /// Adds a `UniqueTypeId` to metadata mapping to the `TypeMap`. The method will
-    /// fail if the mapping already exists.
-    fn register_unique_id_with_metadata(
-        &mut self,
-        unique_type_id: UniqueTypeId,
-        metadata: &'ll DIType,
-    ) {
-        if self
-            .unique_id_to_metadata
-            .insert(unique_type_id, metadata)
-            .is_some()
-        {
-            bug!(
-                "type metadata for unique ID '{}' is already in the `TypeMap`!",
-                self.get_unique_type_id_as_string(unique_type_id)
-            );
-        }
-    }
-
-    fn find_metadata_for_type(&self, type_: Ty<'tcx>) -> Option<&'ll DIType> {
-        self.type_to_metadata.get(&type_).cloned()
-    }
-
-    fn find_metadata_for_unique_id(&self, unique_type_id: UniqueTypeId) -> Option<&'ll DIType> {
-        self.unique_id_to_metadata.get(&unique_type_id).cloned()
-    }
-
-    /// Gets the string representation of a `UniqueTypeId`. This method will fail if
-    /// the ID is unknown.
-    fn get_unique_type_id_as_string(&self, unique_type_id: UniqueTypeId) -> &str {
-        self.unique_id_interner.get(unique_type_id)
-    }
-
-    /// Gets the `UniqueTypeId` for the given type. If the `UniqueTypeId` for the given
-    /// type has been requested before, this is just a table lookup. Otherwise, an
-    /// ID will be generated and stored for later lookup.
-    fn get_unique_type_id_of_type<'a>(
-        &mut self,
-        cx: &CodegenCx<'a, 'tcx>,
-        type_: Ty<'tcx>,
-    ) -> UniqueTypeId {
-        // Let's see if we already have something in the cache.
-        if let Some(unique_type_id) = self.type_to_unique_id.get(&type_).cloned() {
-            return unique_type_id;
-        }
-        // If not, generate one.
-
-        // The hasher we are using to generate the UniqueTypeId. We want
-        // something that provides more than the 64 bits of the DefaultHasher.
-        let mut hasher = StableHasher::new();
-        let mut hcx = cx.tcx.create_stable_hashing_context();
-        let type_ = cx.tcx.erase_regions(type_);
-        hcx.while_hashing_spans(false, |hcx| {
-            hcx.with_node_id_hashing_mode(NodeIdHashingMode::HashDefPath, |hcx| {
-                type_.hash_stable(hcx, &mut hasher);
-            });
-        });
-        let unique_type_id = hasher.finish::<Fingerprint>().to_hex();
-
-        let key = self.unique_id_interner.intern(&unique_type_id);
-        self.type_to_unique_id.insert(type_, key);
-
-        key
-    }
-
-    /// Gets the `UniqueTypeId` for an enum variant. Enum variants are not really
-    /// types of their own, so they need special handling. We still need a
-    /// `UniqueTypeId` for them, since to debuginfo they *are* real types.
-    fn get_unique_type_id_of_enum_variant<'a>(
-        &mut self,
-        cx: &CodegenCx<'a, 'tcx>,
-        enum_type: Ty<'tcx>,
-        variant_name: &str,
-    ) -> UniqueTypeId {
-        let enum_type_id = self.get_unique_type_id_of_type(cx, enum_type);
-        let enum_variant_type_id = format!(
-            "{}::{}",
-            self.get_unique_type_id_as_string(enum_type_id),
-            variant_name
-        );
-        self.unique_id_interner.intern(&enum_variant_type_id)
-    }
-
-    /// Gets the unique type ID string for an enum variant part.
-    /// Variant parts are not types and shouldn't really have their own ID,
-    /// but it makes `set_members_of_composite_type()` simpler.
-    fn get_unique_type_id_str_of_enum_variant_part(
-        &mut self,
-        enum_type_id: UniqueTypeId,
-    ) -> String {
-        format!(
-            "{}_variant_part",
-            self.get_unique_type_id_as_string(enum_type_id)
-        )
-    }
-}
-
-/// A description of some recursive type. It can either be already finished (as
-/// with `FinalMetadata`) or it is not yet finished, but contains all information
-/// needed to generate the missing parts of the description. See the
-/// documentation section on Recursive Types at the top of this file for more
-/// information.
-enum RecursiveTypeDescription<'ll, 'tcx> {
-    UnfinishedMetadata {
-        unfinished_type: Ty<'tcx>,
-        unique_type_id: UniqueTypeId,
-        metadata_stub: &'ll DICompositeType,
-        member_holding_stub: &'ll DICompositeType,
-        member_description_factory: MemberDescriptionFactory<'ll, 'tcx>,
-    },
-    FinalMetadata(&'ll DICompositeType),
-}
-
-use RecursiveTypeDescription::*;
-
-use super::namespace::mangled_name_of_instance;
-use super::util::debug_context;
-use super::CrateDebugContext;
-
-fn create_and_register_recursive_type_forward_declaration<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    unfinished_type: Ty<'tcx>,
-    unique_type_id: UniqueTypeId,
-    metadata_stub: &'ll DICompositeType,
-    member_holding_stub: &'ll DICompositeType,
-    member_description_factory: MemberDescriptionFactory<'ll, 'tcx>,
-) -> RecursiveTypeDescription<'ll, 'tcx> {
-    // Insert the stub into the `TypeMap` in order to allow for recursive references.
-    let mut type_map = debug_context(cx).type_map.borrow_mut();
-    type_map.register_unique_id_with_metadata(unique_type_id, metadata_stub);
-    type_map.register_type_with_metadata(unfinished_type, metadata_stub);
-
-    UnfinishedMetadata {
-        unfinished_type,
-        unique_type_id,
-        metadata_stub,
-        member_description_factory,
-        member_holding_stub,
-    }
-}
+pub(super) const UNKNOWN_LINE_NUMBER: c_uint = 0;
+pub(super) const UNKNOWN_COLUMN_NUMBER: c_uint = 0;
 
-impl<'ll, 'tcx> RecursiveTypeDescription<'ll, 'tcx> {
-    /// Finishes up the description of the type in question (mostly by providing
-    /// descriptions of the fields of the given type) and returns the final type
-    /// metadata.
-    fn finalize(&self, cx: &CodegenCx<'ll, 'tcx>) -> MetadataCreationResult<'ll> {
-        match *self {
-            FinalMetadata(metadata) => MetadataCreationResult::new(metadata, false),
-            UnfinishedMetadata {
-                unfinished_type,
-                unique_type_id,
-                metadata_stub,
-                member_holding_stub,
-                ref member_description_factory,
-            } => {
-                // Make sure that we have a forward declaration of the type in
-                // the TypeMap so that recursive references are possible. This
-                // will always be the case if the RecursiveTypeDescription has
-                // been properly created through the
-                // `create_and_register_recursive_type_forward_declaration()`
-                // function.
-                {
-                    let type_map = debug_context(cx).type_map.borrow();
-                    if type_map
-                        .find_metadata_for_unique_id(unique_type_id)
-                        .is_none()
-                        || type_map.find_metadata_for_type(unfinished_type).is_none()
-                    {
-                        bug!(
-                            "Forward declaration of potentially recursive type \
-                              '{:?}' was not found in TypeMap!",
-                            unfinished_type
-                        );
-                    }
-                }
+const NO_SCOPE_METADATA: Option<&DIScope> = None;
+/// A function that returns an empty list of generic parameter debuginfo nodes.
+const NO_GENERICS: for<'ll> fn(&CodegenCx<'ll, '_>) -> SmallVec<&'ll DIType> = |_| SmallVec::new();
 
-                // ... then create the member descriptions ...
-                let member_descriptions = member_description_factory.create_member_descriptions(cx);
+// SmallVec is used quite a bit in this module, so create a shorthand.
+// The actual number of elements is not so important.
+pub type SmallVec<T> = smallvec::SmallVec<[T; 16]>;
 
-                // ... and attach them to the stub to complete it.
-                set_members_of_composite_type(
-                    cx,
-                    unfinished_type,
-                    member_holding_stub,
-                    member_descriptions,
-                    None,
-                );
-                MetadataCreationResult::new(metadata_stub, true)
-            }
-        }
-    }
-}
+mod enums;
+mod type_map;
 
-/// Returns from the enclosing function if the type metadata with the given
+/// Returns from the enclosing function if the type debuginfo node with the given
 /// unique ID can be found in the type map.
-macro_rules! return_if_metadata_created_in_meantime {
+macro_rules! return_if_di_node_created_in_meantime {
     ($cx: expr, $unique_type_id: expr) => {
-        if let Some(metadata) = debug_context($cx)
+        if let Some(di_node) = debug_context($cx)
             .type_map
-            .borrow()
-            .find_metadata_for_unique_id($unique_type_id)
+            .di_node_for_unique_id($unique_type_id)
         {
-            return MetadataCreationResult::new(metadata, true);
+            return DINodeCreationResult::new(di_node, true);
         }
     };
 }
 
-fn fixed_vec_metadata<'ll, 'tcx>(
+/// Extract size and alignment from a TyAndLayout.
+#[inline]
+fn size_and_align_of(ty_and_layout: TyAndLayout<'_>) -> (Size, Align) {
+    (ty_and_layout.size, ty_and_layout.align.abi)
+}
+
+/// Creates debuginfo for a fixed size array (e.g. `[u64; 123]`).
+/// For slices (that is, "arrays" of unknown size) use [build_slice_type_di_node].
+fn build_fixed_size_array_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
-    unique_type_id: UniqueTypeId,
-    array_or_slice_type: Ty<'tcx>,
-    element_type: Ty<'tcx>,
-    span: Span,
-) -> MetadataCreationResult<'ll> {
-    let element_type_metadata = type_metadata(cx, element_type, span);
+    unique_type_id: UniqueTypeId<'tcx>,
+    array_type: Ty<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    let ty::Array(element_type, len) = array_type.kind() else {
+        bug!(
+            "build_fixed_size_array_di_node() called with non-ty::Array type `{:?}`",
+            array_type
+        )
+    };
 
-    return_if_metadata_created_in_meantime!(cx, unique_type_id);
+    let element_type_di_node = type_di_node(cx, *element_type);
 
-    let (size, align) = cx.size_and_align_of(array_or_slice_type);
+    return_if_di_node_created_in_meantime!(cx, unique_type_id);
 
-    let upper_bound = match array_or_slice_type.kind() {
-        ty::Array(_, len) => len.eval_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong,
-        _ => -1,
-    };
+    let (size, align) = cx.size_and_align_of(array_type);
+
+    let upper_bound = len
+        .try_to_target_usize(cx.tcx)
+        .expect("expected monomorphic const in codegen") as c_longlong;
 
     let subrange = unsafe {
         Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(
@@ -410,90 +122,190 @@ fn fixed_vec_metadata<'ll, 'tcx>(
     };
 
     let subscripts = create_DIArray(DIB(cx), &[subrange]);
-    let metadata = unsafe {
+    let di_node = unsafe {
         llvm::LLVMRustDIBuilderCreateArrayType(
             DIB(cx),
             size.bits(),
             align.bits() as u32,
-            element_type_metadata,
+            element_type_di_node,
             subscripts,
         )
     };
 
-    MetadataCreationResult::new(metadata, false)
+    DINodeCreationResult::new(di_node, false)
 }
 
-fn vec_slice_metadata<'ll, 'tcx>(
+/// Creates debuginfo for built-in pointer-like things:
+///
+///  - ty::Ref
+///  - ty::RawPtr
+///  - ty::Adt in the case it's Box
+///
+/// At some point we might want to remove the special handling of Box
+/// and treat it the same as other smart pointers (like Rc, Arc, ...).
+fn build_pointer_or_reference_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
-    slice_ptr_type: Ty<'tcx>,
-    element_type: Ty<'tcx>,
-    unique_type_id: UniqueTypeId,
-    span: Span,
-) -> MetadataCreationResult<'ll> {
-    let data_ptr_type = cx.tcx.mk_imm_ptr(element_type);
-
-    let data_ptr_metadata = type_metadata(cx, data_ptr_type, span);
-
-    return_if_metadata_created_in_meantime!(cx, unique_type_id);
-
-    let slice_type_name = compute_debuginfo_type_name(cx.tcx, slice_ptr_type, true);
-
-    let (pointer_size, pointer_align) = cx.size_and_align_of(data_ptr_type);
-    let (usize_size, usize_align) = cx.size_and_align_of(cx.tcx.types.usize);
-
-    let member_descriptions = vec![
-        MemberDescription {
-            name: "data_ptr".to_owned(),
-            type_metadata: data_ptr_metadata,
-            offset: Size::ZERO,
-            size: pointer_size,
-            align: pointer_align,
-            flags: DIFlags::FlagZero,
-            discriminant: None,
-            source_info: None,
-        },
-        MemberDescription {
-            name: "length".to_owned(),
-            type_metadata: type_metadata(cx, cx.tcx.types.usize, span),
-            offset: pointer_size,
-            size: usize_size,
-            align: usize_align,
-            flags: DIFlags::FlagZero,
-            discriminant: None,
-            source_info: None,
-        },
-    ];
+    ptr_type: Ty<'tcx>,
+    pointee_type: Ty<'tcx>,
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    // The debuginfo generated by this function is only valid if `ptr_type` is really just
+    // a (wide) pointer. Make sure it is not called for e.g. `Box<T, NonZSTAllocator>`.
+    assert_eq!(
+        cx.size_and_align_of(ptr_type),
+        cx.size_and_align_of(Ty::new_mut_ptr(cx.tcx, pointee_type))
+    );
 
-    let file_metadata = unknown_file_metadata(cx);
+    let pointee_type_di_node = type_di_node(cx, pointee_type);
 
-    let metadata = composite_type_metadata(
-        cx,
-        slice_ptr_type,
-        &slice_type_name[..],
-        unique_type_id,
-        member_descriptions,
-        NO_SCOPE_METADATA,
-        file_metadata,
-        span,
-    );
-    MetadataCreationResult::new(metadata, false)
+    return_if_di_node_created_in_meantime!(cx, unique_type_id);
+
+    let data_layout = &cx.tcx.data_layout;
+    let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true);
+
+    match wide_pointer_kind(cx, pointee_type) {
+        None => {
+            // This is a thin pointer. Create a regular pointer type and give it the correct name.
+            assert_eq!(
+                (data_layout.pointer_size, data_layout.pointer_align.abi),
+                cx.size_and_align_of(ptr_type),
+                "ptr_type={ptr_type}, pointee_type={pointee_type}",
+            );
+
+            let di_node = unsafe {
+                llvm::LLVMRustDIBuilderCreatePointerType(
+                    DIB(cx),
+                    pointee_type_di_node,
+                    data_layout.pointer_size.bits(),
+                    data_layout.pointer_align.abi.bits() as u32,
+                    CString::new(ptr_type_debuginfo_name).unwrap().as_ptr(),
+                )
+            };
+
+            DINodeCreationResult {
+                di_node,
+                already_stored_in_typemap: false,
+            }
+        }
+        Some(wide_pointer_kind) => {
+            type_map::build_type_with_children(
+                cx,
+                type_map::stub(
+                    cx,
+                    Stub::Struct,
+                    unique_type_id,
+                    &ptr_type_debuginfo_name,
+                    None,
+                    cx.size_and_align_of(ptr_type),
+                    NO_SCOPE_METADATA,
+                    DIFlags::FlagZero,
+                ),
+                |cx, owner| {
+                    // FIXME: If this wide pointer is a `Box` then we don't want to use its
+                    //        type layout and instead use the layout of the raw pointer inside
+                    //        of it.
+                    //        The proper way to handle this is to not treat Box as a pointer
+                    //        at all and instead emit regular struct debuginfo for it. We just
+                    //        need to make sure that we don't break existing debuginfo consumers
+                    //        by doing that (at least not without a warning period).
+                    let layout_type = if ptr_type.is_box() {
+                        // The assertion at the start of this function ensures we have a ZST
+                        // allocator. We'll make debuginfo "skip" all ZST allocators, not just the
+                        // default allocator.
+                        Ty::new_mut_ptr(cx.tcx, pointee_type)
+                    } else {
+                        ptr_type
+                    };
+
+                    let layout = cx.layout_of(layout_type);
+                    let addr_field = layout.field(cx, abi::WIDE_PTR_ADDR);
+                    let extra_field = layout.field(cx, abi::WIDE_PTR_EXTRA);
+
+                    let (addr_field_name, extra_field_name) = match wide_pointer_kind {
+                        WidePtrKind::Dyn => ("pointer", "vtable"),
+                        WidePtrKind::Slice => ("data_ptr", "length"),
+                    };
+
+                    assert_eq!(abi::WIDE_PTR_ADDR, 0);
+                    assert_eq!(abi::WIDE_PTR_EXTRA, 1);
+
+                    // The data pointer type is a regular, thin pointer, regardless of whether this
+                    // is a slice or a trait object.
+                    let data_ptr_type_di_node = unsafe {
+                        llvm::LLVMRustDIBuilderCreatePointerType(
+                            DIB(cx),
+                            pointee_type_di_node,
+                            addr_field.size.bits(),
+                            addr_field.align.abi.bits() as u32,
+                            // 0, // Ignore DWARF address space.
+                            c"".as_ptr(),
+                        )
+                    };
+
+                    smallvec![
+                        build_field_di_node(
+                            cx,
+                            owner,
+                            addr_field_name,
+                            (addr_field.size, addr_field.align.abi),
+                            layout.fields.offset(abi::WIDE_PTR_ADDR),
+                            DIFlags::FlagZero,
+                            data_ptr_type_di_node,
+                            None,
+                        ),
+                        build_field_di_node(
+                            cx,
+                            owner,
+                            extra_field_name,
+                            (extra_field.size, extra_field.align.abi),
+                            layout.fields.offset(abi::WIDE_PTR_EXTRA),
+                            DIFlags::FlagZero,
+                            type_di_node(cx, extra_field.ty),
+                            None,
+                        ),
+                    ]
+                },
+                NO_GENERICS,
+            )
+        }
+    }
 }
 
-fn subroutine_type_metadata<'ll, 'tcx>(
+fn build_subroutine_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
-    unique_type_id: UniqueTypeId,
-    signature: ty::PolyFnSig<'tcx>,
-    span: Span,
-) -> MetadataCreationResult<'ll> {
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    // It's possible to create a self-referential type in Rust by using 'impl trait':
+    //
+    // fn foo() -> impl Copy { foo }
+    //
+    // Unfortunately LLVM's API does not allow us to create recursive subroutine types.
+    // In order to work around that restriction we place a marker type in the type map,
+    // before creating the actual type. If the actual type is recursive, it will hit the
+    // marker type. So we end up with a type that looks like
+    //
+    // fn foo() -> <recursive_type>
+    //
+    // Once that is created, we replace the marker in the typemap with the actual type.
+    debug_context(cx)
+        .type_map
+        .unique_id_to_di_node
+        .borrow_mut()
+        .insert(unique_type_id, recursion_marker_type_di_node(cx));
+
+    let fn_ty = unique_type_id.expect_ty();
     let signature = cx
         .tcx
-        .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), signature);
+        .normalize_erasing_late_bound_regions(cx.typing_env(), fn_ty.fn_sig(cx.tcx));
 
-    let signature_metadata: Vec<_> = iter::once(
+    let signature_di_nodes: SmallVec<_> = iter::once(
         // return type
         match signature.output().kind() {
-            ty::Tuple(tys) if tys.is_empty() => None,
-            _ => Some(type_metadata(cx, signature.output(), span)),
+            ty::Tuple(tys) if tys.is_empty() => {
+                // this is a "void" function
+                None
+            }
+            _ => Some(type_di_node(cx, signature.output())),
         },
     )
     .chain(
@@ -501,490 +313,488 @@ fn subroutine_type_metadata<'ll, 'tcx>(
         signature
             .inputs()
             .iter()
-            .map(|argument_type| Some(type_metadata(cx, argument_type, span))),
+            .map(|&argument_type| Some(type_di_node(cx, argument_type))),
     )
     .collect();
 
-    return_if_metadata_created_in_meantime!(cx, unique_type_id);
-
-    MetadataCreationResult::new(
-        unsafe {
-            llvm::LLVMRustDIBuilderCreateSubroutineType(
-                DIB(cx),
-                create_DIArray(DIB(cx), &signature_metadata[..]),
-            )
-        },
-        false,
-    )
-}
+    debug_context(cx)
+        .type_map
+        .unique_id_to_di_node
+        .borrow_mut()
+        .remove(&unique_type_id);
 
-// FIXME(1563): This is all a bit of a hack because 'trait pointer' is an ill-
-// defined concept. For the case of an actual trait pointer (i.e., `Box<Trait>`,
-// `&Trait`), `trait_object_type` should be the whole thing (e.g, `Box<Trait>`) and
-// `trait_type` should be the actual trait (e.g., `Trait`). Where the trait is part
-// of a DST struct, there is no `trait_object_type` and the results of this
-// function will be a little bit weird.
-fn trait_pointer_metadata<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    trait_type: Ty<'tcx>,
-    trait_object_type: Option<Ty<'tcx>>,
-    unique_type_id: UniqueTypeId,
-) -> &'ll DIType {
-    // The implementation provided here is a stub. It makes sure that the trait
-    // type is assigned the correct name, size, namespace, and source location.
-    // However, it does not describe the trait's methods.
-
-    let (containing_scope, trait_type_name) = match trait_object_type {
-        Some(trait_object_type) => match trait_object_type.kind() {
-            ty::Adt(def, _) => (
-                Some(get_namespace_for_item(cx, def.did)),
-                compute_debuginfo_type_name(cx.tcx, trait_object_type, false),
-            ),
-            ty::RawPtr(_) | ty::Ref(..) => (
-                NO_SCOPE_METADATA,
-                compute_debuginfo_type_name(cx.tcx, trait_object_type, true),
-            ),
-            _ => {
-                bug!(
-                    "debuginfo: unexpected trait-object type in \
-                      trait_pointer_metadata(): {:?}",
-                    trait_object_type
-                );
-            }
-        },
+    let fn_di_node = unsafe {
+        llvm::LLVMRustDIBuilderCreateSubroutineType(
+            DIB(cx),
+            create_DIArray(DIB(cx), &signature_di_nodes[..]),
+        )
+    };
 
-        // No object type, use the trait type directly (no scope here since the type
-        // will be wrapped in the dyn$ synthetic type).
-        None => (
-            NO_SCOPE_METADATA,
-            compute_debuginfo_type_name(cx.tcx, trait_type, true),
+    // This is actually a function pointer, so wrap it in pointer DI.
+    let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false);
+    let (size, align) = match fn_ty.kind() {
+        ty::FnDef(..) => (0, 1),
+        ty::FnPtr(..) => (
+            cx.tcx.data_layout.pointer_size.bits(),
+            cx.tcx.data_layout.pointer_align.abi.bits() as u32,
         ),
+        _ => unreachable!(),
+    };
+    let di_node = unsafe {
+        llvm::LLVMRustDIBuilderCreatePointerType(
+            DIB(cx),
+            fn_di_node,
+            size,
+            align,
+            CString::new(name).unwrap().as_ptr(),
+        )
     };
 
-    let file_metadata = unknown_file_metadata(cx);
-
-    let layout = cx.layout_of(cx.tcx.mk_mut_ptr(trait_type));
-
-    assert_eq!(crate::abi::FAT_PTR_ADDR, 0);
-    assert_eq!(crate::abi::FAT_PTR_EXTRA, 1);
+    DINodeCreationResult::new(di_node, false)
+}
 
-    let data_ptr_field = layout.field(cx, 0);
-    let vtable_field = layout.field(cx, 1);
-    let member_descriptions = vec![
-        MemberDescription {
-            name: "pointer".to_owned(),
-            type_metadata: type_metadata(
+/// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs
+/// we with the correct type name (e.g. "dyn SomeTrait<Foo, Item=u32> + Sync").
+fn build_dyn_type_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    dyn_type: Ty<'tcx>,
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    if let ty::Dynamic(..) = dyn_type.kind() {
+        let type_name = compute_debuginfo_type_name(cx.tcx, dyn_type, true);
+        type_map::build_type_with_children(
+            cx,
+            type_map::stub(
                 cx,
-                cx.tcx.mk_mut_ptr(cx.tcx.types.u8),
-                rustc_span::DUMMY_SP,
+                Stub::Struct,
+                unique_type_id,
+                &type_name,
+                None,
+                cx.size_and_align_of(dyn_type),
+                NO_SCOPE_METADATA,
+                DIFlags::FlagZero,
             ),
-            offset: layout.fields.offset(0),
-            size: data_ptr_field.size,
-            align: data_ptr_field.align.abi,
-            flags: DIFlags::FlagArtificial,
-            discriminant: None,
-            source_info: None,
-        },
-        MemberDescription {
-            name: "vtable".to_owned(),
-            type_metadata: type_metadata(cx, vtable_field.ty, rustc_span::DUMMY_SP),
-            offset: layout.fields.offset(1),
-            size: vtable_field.size,
-            align: vtable_field.align.abi,
-            flags: DIFlags::FlagArtificial,
-            discriminant: None,
-            source_info: None,
-        },
-    ];
-
-    composite_type_metadata(
-        cx,
-        trait_object_type.unwrap_or(trait_type),
-        &trait_type_name[..],
-        unique_type_id,
-        member_descriptions,
-        containing_scope,
-        file_metadata,
-        rustc_span::DUMMY_SP,
-    )
+            |_, _| smallvec![],
+            NO_GENERICS,
+        )
+    } else {
+        bug!(
+            "Only ty::Dynamic is valid for build_dyn_type_di_node(). Found {:?} instead.",
+            dyn_type
+        )
+    }
 }
 
-pub(crate) fn type_metadata<'ll, 'tcx>(
+/// Create debuginfo for `[T]` and `str`. These are unsized.
+///
+/// NOTE: We currently emit just emit the debuginfo for the element type here
+/// (i.e. `T` for slices and `u8` for `str`), so that we end up with
+/// `*const T` for the `data_ptr` field of the corresponding wide-pointer
+/// debuginfo of `&[T]`.
+///
+/// It would be preferable and more accurate if we emitted a DIArray of T
+/// without an upper bound instead. That is, LLVM already supports emitting
+/// debuginfo of arrays of unknown size. But GDB currently seems to end up
+/// in an infinite loop when confronted with such a type.
+///
+/// As a side effect of the current encoding every instance of a type like
+/// `struct Foo { unsized_field: [u8] }` will look like
+/// `struct Foo { unsized_field: u8 }` in debuginfo. If the length of the
+/// slice is zero, then accessing `unsized_field` in the debugger would
+/// result in an out-of-bounds access.
+fn build_slice_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
-    t: Ty<'tcx>,
-    usage_site_span: Span,
-) -> &'ll DIType {
-    // Get the unique type ID of this type.
-    let unique_type_id = {
-        let mut type_map = debug_context(cx).type_map.borrow_mut();
-        // First, try to find the type in `TypeMap`. If we have seen it before, we
-        // can exit early here.
-        match type_map.find_metadata_for_type(t) {
-            Some(metadata) => {
-                return metadata;
-            }
-            None => {
-                // The Ty is not in the `TypeMap` but maybe we have already seen
-                // an equivalent type (e.g., only differing in region arguments).
-                // In order to find out, generate the unique type ID and look
-                // that up.
-                let unique_type_id = type_map.get_unique_type_id_of_type(cx, t);
-                match type_map.find_metadata_for_unique_id(unique_type_id) {
-                    Some(metadata) => {
-                        // There is already an equivalent type in the TypeMap.
-                        // Register this Ty as an alias in the cache and
-                        // return the cached metadata.
-                        type_map.register_type_with_metadata(t, metadata);
-                        return metadata;
-                    }
-                    None => {
-                        // There really is no type metadata for this type, so
-                        // proceed by creating it.
-                        unique_type_id
-                    }
-                }
-            }
+    slice_type: Ty<'tcx>,
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    let element_type = match slice_type.kind() {
+        ty::Slice(element_type) => *element_type,
+        ty::Str => cx.tcx.types.u8,
+        _ => {
+            bug!(
+                "Only ty::Slice is valid for build_slice_type_di_node(). Found {:?} instead.",
+                slice_type
+            )
         }
     };
 
-    let ptr_metadata = |ty: Ty<'tcx>| match *ty.kind() {
-        ty::Slice(typ) => Ok(vec_slice_metadata(
-            cx,
-            t,
-            typ,
-            unique_type_id,
-            usage_site_span,
-        )),
-        ty::Str => Ok(vec_slice_metadata(
-            cx,
-            t,
-            cx.tcx.types.u8,
-            unique_type_id,
-            usage_site_span,
-        )),
-        ty::Dynamic(..) => Ok(MetadataCreationResult::new(
-            trait_pointer_metadata(cx, ty, Some(t), unique_type_id),
-            false,
-        )),
-        _ => {
-            let pointee_metadata = type_metadata(cx, ty, usage_site_span);
+    let element_type_di_node = type_di_node(cx, element_type);
+    return_if_di_node_created_in_meantime!(cx, unique_type_id);
+    DINodeCreationResult {
+        di_node: element_type_di_node,
+        already_stored_in_typemap: false,
+    }
+}
 
-            if let Some(metadata) = debug_context(cx)
-                .type_map
-                .borrow()
-                .find_metadata_for_unique_id(unique_type_id)
-            {
-                return Err(metadata);
-            }
+pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType {
+    // Get the unique type ID of this type.
+    let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t);
 
-            Ok(MetadataCreationResult::new(
-                pointer_type_metadata(cx, t, pointee_metadata),
-                false,
-            ))
-        }
-    };
+    if let Some(existing_di_node) = debug_context(cx)
+        .type_map
+        .di_node_for_unique_id(unique_type_id)
+    {
+        return existing_di_node;
+    }
 
-    let MetadataCreationResult {
-        metadata,
+    debug!("type_di_node: {:?} kind: {:?}", t, t.kind());
+
+    let DINodeCreationResult {
+        di_node,
         already_stored_in_typemap,
     } = match *t.kind() {
         ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => {
-            MetadataCreationResult::new(basic_type_metadata(cx, t), false)
-        }
-        ty::Tuple(elements) if elements.is_empty() => {
-            MetadataCreationResult::new(basic_type_metadata(cx, t), false)
+            build_basic_type_di_node(cx, t)
         }
-        ty::Array(typ, _) | ty::Slice(typ) => {
-            fixed_vec_metadata(cx, unique_type_id, t, typ, usage_site_span)
+        ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t),
+        ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t),
+        ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id),
+        ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id),
+        ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id),
+        ty::RawPtr(pointee_type, _) | ty::Ref(_, pointee_type, _) => {
+            build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id)
         }
-        ty::Str => fixed_vec_metadata(cx, unique_type_id, t, cx.tcx.types.i8, usage_site_span),
-        ty::Dynamic(..) => {
-            MetadataCreationResult::new(trait_pointer_metadata(cx, t, None, unique_type_id), false)
-        }
-        ty::Foreign(..) => {
-            MetadataCreationResult::new(foreign_type_metadata(cx, t, unique_type_id), false)
-        }
-        ty::RawPtr(ty::TypeAndMut { ty, .. }) | ty::Ref(_, ty, _) => match ptr_metadata(ty) {
-            Ok(res) => res,
-            Err(metadata) => return metadata,
-        },
-        ty::Adt(def, _) if def.is_box() => match ptr_metadata(t.boxed_ty()) {
-            Ok(res) => res,
-            Err(metadata) => return metadata,
-        },
-        ty::FnDef(..) | ty::FnPtr(_) => {
-            if let Some(metadata) = debug_context(cx)
-                .type_map
-                .borrow()
-                .find_metadata_for_unique_id(unique_type_id)
-            {
-                return metadata;
-            }
-
-            // It's possible to create a self-referential
-            // type in Rust by using 'impl trait':
-            //
-            // fn foo() -> impl Copy { foo }
-            //
-            // See `TypeMap::remove_type` for more detals
-            // about the workaround.
-
-            let temp_type = {
-                unsafe {
-                    // The choice of type here is pretty arbitrary -
-                    // anything reading the debuginfo for a recursive
-                    // type is going to see *something* weird - the only
-                    // question is what exactly it will see.
-                    let name = "<recur_type>\0";
-                    llvm::LLVMRustDIBuilderCreateBasicType(
-                        DIB(cx),
-                        name.as_ptr().cast(),
-                        cx.size_of(t).bits(),
-                        DW_ATE_unsigned,
-                    )
-                }
-            };
-
-            let type_map = &debug_context(cx).type_map;
-            type_map
-                .borrow_mut()
-                .register_type_with_metadata(t, temp_type);
-
-            let fn_metadata =
-                subroutine_type_metadata(cx, unique_type_id, t.fn_sig(cx.tcx), usage_site_span)
-                    .metadata;
-
-            type_map.borrow_mut().remove_type(t);
-
-            // This is actually a function pointer, so wrap it in pointer DI.
-            MetadataCreationResult::new(pointer_type_metadata(cx, t, fn_metadata), false)
-        }
-        ty::Closure(def_id, substs) => {
-            let upvar_tys: Vec<_> = substs.as_closure().upvar_tys().collect();
-            let containing_scope = get_namespace_for_item(cx, def_id);
-            prepare_tuple_metadata(
-                cx,
-                t,
-                &upvar_tys,
-                unique_type_id,
-                usage_site_span,
-                Some(containing_scope),
-            )
-            .finalize(cx)
-        }
-        ty::Generator(def_id, substs, _) => {
-            let upvar_tys: Vec<_> = substs
-                .as_generator()
-                .prefix_tys()
-                .map(|t| cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t))
-                .collect();
-            prepare_enum_metadata(cx, t, def_id, unique_type_id, usage_site_span, upvar_tys)
-                .finalize(cx)
+        // Some `Box` are newtyped pointers, make debuginfo aware of that.
+        // Only works if the allocator argument is a 1-ZST and hence irrelevant for layout
+        // (or if there is no allocator argument).
+        ty::Adt(def, args)
+            if def.is_box()
+                && args
+                    .get(1)
+                    .map_or(true, |arg| cx.layout_of(arg.expect_ty()).is_1zst()) =>
+        {
+            build_pointer_or_reference_di_node(cx, t, t.expect_boxed_ty(), unique_type_id)
         }
+        ty::FnDef(..) | ty::FnPtr(..) => build_subroutine_type_di_node(cx, unique_type_id),
+        ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id),
+        ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id),
+        ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id),
         ty::Adt(def, ..) => match def.adt_kind() {
-            AdtKind::Struct => {
-                prepare_struct_metadata(cx, t, unique_type_id, usage_site_span).finalize(cx)
-            }
-            AdtKind::Union => {
-                prepare_union_metadata(cx, t, unique_type_id, usage_site_span).finalize(cx)
-            }
-            AdtKind::Enum => {
-                prepare_enum_metadata(cx, t, def.did, unique_type_id, usage_site_span, vec![])
-                    .finalize(cx)
-            }
+            AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id),
+            AdtKind::Union => build_union_type_di_node(cx, unique_type_id),
+            AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id),
         },
-        ty::Tuple(elements) => {
-            let tys: Vec<_> = elements.iter().map(|k| k.expect_ty()).collect();
-            prepare_tuple_metadata(
-                cx,
-                t,
-                &tys,
-                unique_type_id,
-                usage_site_span,
-                NO_SCOPE_METADATA,
-            )
-            .finalize(cx)
-        }
-        // Type parameters from polymorphized functions.
-        ty::Param(_) => MetadataCreationResult::new(param_type_metadata(cx, t), false),
-        _ => bug!("debuginfo: unexpected type in type_metadata: {:?}", t),
+        ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id),
+        _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t),
     };
 
     {
-        let mut type_map = debug_context(cx).type_map.borrow_mut();
-
         if already_stored_in_typemap {
-            // Also make sure that we already have a `TypeMap` entry for the unique type ID.
-            let metadata_for_uid = match type_map.find_metadata_for_unique_id(unique_type_id) {
-                Some(metadata) => metadata,
+            // Make sure that we really do have a `TypeMap` entry for the unique type ID.
+            let di_node_for_uid = match debug_context(cx)
+                .type_map
+                .di_node_for_unique_id(unique_type_id)
+            {
+                Some(di_node) => di_node,
                 None => {
-                    span_bug!(
-                        usage_site_span,
-                        "expected type metadata for unique \
-                               type ID '{}' to already be in \
+                    bug!(
+                        "expected type debuginfo node for unique \
+                               type ID '{:?}' to already be in \
                                the `debuginfo::TypeMap` but it \
-                               was not. (Ty = {})",
-                        type_map.get_unique_type_id_as_string(unique_type_id),
-                        t
+                               was not.",
+                        unique_type_id,
                     );
                 }
             };
 
-            match type_map.find_metadata_for_type(t) {
-                Some(metadata) => {
-                    if metadata != metadata_for_uid {
-                        span_bug!(
-                            usage_site_span,
-                            "mismatch between `Ty` and \
-                                   `UniqueTypeId` maps in \
-                                   `debuginfo::TypeMap`. \
-                                   UniqueTypeId={}, Ty={}",
-                            type_map.get_unique_type_id_as_string(unique_type_id),
-                            t
-                        );
-                    }
-                }
-                None => {
-                    type_map.register_type_with_metadata(t, metadata);
-                }
-            }
+            assert_eq!(di_node_for_uid as *const _, di_node as *const _);
         } else {
-            type_map.register_type_with_metadata(t, metadata);
-            type_map.register_unique_id_with_metadata(unique_type_id, metadata);
+            debug_context(cx).type_map.insert(unique_type_id, di_node);
         }
     }
 
-    metadata
+    di_node
 }
 
-pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile {
-    let hash = Some(&source_file.src_hash);
-    let file_name = Some(source_file.name.prefer_remapped().to_string());
-    let directory = if source_file.is_real_file() && !source_file.is_imported() {
-        Some(
-            cx.sess()
-                .opts
-                .working_dir
-                .to_string_lossy(FileNameDisplayPreference::Remapped)
-                .to_string(),
-        )
-    } else {
-        // If the path comes from an upstream crate we assume it has been made
-        // independent of the compiler's working directory one way or another.
-        None
-    };
-    file_metadata_raw(cx, file_name, directory, hash)
+// FIXME(mw): Cache this via a regular UniqueTypeId instead of an extra field in the debug context.
+fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll DIType {
+    *debug_context(cx)
+        .recursion_marker_type
+        .get_or_init(move || {
+            unsafe {
+                // The choice of type here is pretty arbitrary -
+                // anything reading the debuginfo for a recursive
+                // type is going to see *something* weird - the only
+                // question is what exactly it will see.
+                //
+                // FIXME: the name `<recur_type>` does not fit the naming scheme
+                //        of other types.
+                //
+                // FIXME: it might make sense to use an actual pointer type here
+                //        so that debuggers can show the address.
+                let name = "<recur_type>";
+                llvm::LLVMRustDIBuilderCreateBasicType(
+                    DIB(cx),
+                    name.as_c_char_ptr(),
+                    name.len(),
+                    cx.tcx.data_layout.pointer_size.bits(),
+                    dwarf_const::DW_ATE_unsigned,
+                )
+            }
+        })
 }
 
-pub(crate) fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile {
-    file_metadata_raw(cx, None, None, None)
+fn hex_encode(data: &[u8]) -> String {
+    let mut hex_string = String::with_capacity(data.len() * 2);
+    for byte in data.iter() {
+        write!(&mut hex_string, "{byte:02x}").unwrap();
+    }
+    hex_string
 }
 
-fn file_metadata_raw<'ll>(
-    cx: &CodegenCx<'ll, '_>,
-    file_name: Option<String>,
-    directory: Option<String>,
-    _hash: Option<&SourceFileHash>,
-) -> &'ll DIFile {
-    let key = (file_name, directory);
-
-    let mut created_files = debug_context(cx).created_files.borrow_mut();
-    match created_files.entry(key.clone()) {
-        Entry::Occupied(o) => o.get(),
-        Entry::Vacant(v) => {
-            let (file_name, directory) = v.key();
-
-            let file_name = CString::new(file_name.as_deref().unwrap_or("<unknown>")).unwrap();
-            let directory = CString::new(directory.as_deref().unwrap_or("")).unwrap();
-
-            let file_metadata = unsafe {
-                llvm::LLVMRustDIBuilderCreateFile(DIB(cx), file_name.as_ptr(), directory.as_ptr())
-            };
+pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile {
+    let cache_key = Some((source_file.stable_id, source_file.src_hash));
+    return debug_context(cx)
+        .created_files
+        .borrow_mut()
+        .entry(cache_key)
+        .or_insert_with(|| alloc_new_file_metadata(cx, source_file));
+
+    fn alloc_new_file_metadata<'ll>(
+        cx: &CodegenCx<'ll, '_>,
+        source_file: &SourceFile,
+    ) -> &'ll DIFile {
+        debug!(?source_file.name);
+
+        let filename_display_preference = cx
+            .sess()
+            .filename_display_preference(RemapPathScopeComponents::DEBUGINFO);
+
+        use rustc_session::config::RemapPathScopeComponents;
+        let (directory, file_name) = match &source_file.name {
+            FileName::Real(filename) => {
+                let working_directory = &cx.sess().opts.working_dir;
+                debug!(?working_directory);
+
+                if filename_display_preference == FileNameDisplayPreference::Remapped {
+                    let filename = cx
+                        .sess()
+                        .source_map()
+                        .path_mapping()
+                        .to_embeddable_absolute_path(filename.clone(), working_directory);
+
+                    // Construct the absolute path of the file
+                    let abs_path = filename.remapped_path_if_available();
+                    debug!(?abs_path);
+
+                    if let Ok(rel_path) =
+                        abs_path.strip_prefix(working_directory.remapped_path_if_available())
+                    {
+                        // If the compiler's working directory (which also is the DW_AT_comp_dir of
+                        // the compilation unit) is a prefix of the path we are about to emit, then
+                        // only emit the part relative to the working directory. Because of path
+                        // remapping we sometimes see strange things here: `abs_path` might
+                        // actually look like a relative path (e.g.
+                        // `<crate-name-and-version>/src/lib.rs`), so if we emit it without taking
+                        // the working directory into account, downstream tooling will interpret it
+                        // as `<working-directory>/<crate-name-and-version>/src/lib.rs`, which
+                        // makes no sense. Usually in such cases the working directory will also be
+                        // remapped to `<crate-name-and-version>` or some other prefix of the path
+                        // we are remapping, so we end up with
+                        // `<crate-name-and-version>/<crate-name-and-version>/src/lib.rs`.
+                        // By moving the working directory portion into the `directory` part of the
+                        // DIFile, we allow LLVM to emit just the relative path for DWARF, while
+                        // still emitting the correct absolute path for CodeView.
+                        (
+                            working_directory.to_string_lossy(FileNameDisplayPreference::Remapped),
+                            rel_path.to_string_lossy().into_owned(),
+                        )
+                    } else {
+                        ("".into(), abs_path.to_string_lossy().into_owned())
+                    }
+                } else {
+                    let working_directory = working_directory.local_path_if_available();
+                    let filename = filename.local_path_if_available();
 
-            created_files.insert(key, file_metadata);
-            file_metadata
-        }
-    }
-}
+                    debug!(?working_directory, ?filename);
 
-fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType {
-    let (name, encoding) = match t.kind() {
-        ty::Never => ("!", DW_ATE_unsigned),
-        ty::Tuple(elements) if elements.is_empty() => ("()", DW_ATE_unsigned),
-        ty::Bool => ("bool", DW_ATE_boolean),
-        ty::Char => ("char", DW_ATE_unsigned_char),
+                    let abs_path: Cow<'_, _> = if filename.is_absolute() {
+                        filename.into()
+                    } else {
+                        let mut p = PathBuf::new();
+                        p.push(working_directory);
+                        p.push(filename);
+                        p.into()
+                    };
+
+                    if let Ok(rel_path) = abs_path.strip_prefix(working_directory) {
+                        (
+                            working_directory.to_string_lossy(),
+                            rel_path.to_string_lossy().into_owned(),
+                        )
+                    } else {
+                        ("".into(), abs_path.to_string_lossy().into_owned())
+                    }
+                }
+            }
+            other => {
+                debug!(?other);
+                (
+                    "".into(),
+                    other.display(filename_display_preference).to_string(),
+                )
+            }
+        };
+
+        let hash_kind = match source_file.src_hash.kind {
+            rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5,
+            rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1,
+            // NVVM: Llvm 7 does not support either of these checksums
+            rustc_span::SourceFileHashAlgorithm::Sha256
+            | rustc_span::SourceFileHashAlgorithm::Blake3 => llvm::ChecksumKind::None,
+        };
+        let hash_value = hex_encode(source_file.src_hash.hash_bytes());
+
+        let source = cx
+            .sess()
+            .opts
+            .unstable_opts
+            .embed_source
+            .then_some(())
+            .and(source_file.src.as_ref());
+
+        unsafe {
+            llvm::LLVMRustDIBuilderCreateFile(
+                DIB(cx),
+                file_name.as_c_char_ptr(),
+                file_name.len(),
+                directory.as_c_char_ptr(),
+                directory.len(),
+                hash_kind,
+                hash_value.as_c_char_ptr(),
+                hash_value.len(),
+                source.map_or(ptr::null(), |x| x.as_c_char_ptr()),
+                source.map_or(0, |x| x.len()),
+            )
+        }
+    }
+}
+
+pub(crate) fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile {
+    debug_context(cx)
+        .created_files
+        .borrow_mut()
+        .entry(None)
+        .or_insert_with(|| unsafe {
+            let file_name = "<unknown>";
+            let directory = "";
+            let hash_value = "";
+
+            llvm::LLVMRustDIBuilderCreateFile(
+                DIB(cx),
+                file_name.as_c_char_ptr(),
+                file_name.len(),
+                directory.as_c_char_ptr(),
+                directory.len(),
+                llvm::ChecksumKind::None,
+                hash_value.as_c_char_ptr(),
+                hash_value.len(),
+                ptr::null(),
+                0,
+            )
+        })
+}
+
+fn build_basic_type_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    t: Ty<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    debug!("build_basic_type_di_node: {:?}", t);
+
+    use dwarf_const::{DW_ATE_UTF, DW_ATE_boolean, DW_ATE_float, DW_ATE_signed, DW_ATE_unsigned};
+
+    let (name, encoding) = match t.kind() {
+        ty::Never => ("!", DW_ATE_unsigned),
+        ty::Tuple(elements) if elements.is_empty() => ("()", DW_ATE_unsigned),
+        ty::Bool => ("bool", DW_ATE_boolean),
+        ty::Char => ("char", DW_ATE_UTF),
         ty::Int(int_ty) => (int_ty.name_str(), DW_ATE_signed),
         ty::Uint(uint_ty) => (uint_ty.name_str(), DW_ATE_unsigned),
         ty::Float(float_ty) => (float_ty.name_str(), DW_ATE_float),
-        _ => bug!("debuginfo::basic_type_metadata - `t` is invalid type"),
+        _ => bug!("debuginfo::build_basic_type_di_node - `t` is invalid type"),
     };
-    let name = CString::new(name).unwrap();
 
-    unsafe {
+    let ty_di_node = unsafe {
         llvm::LLVMRustDIBuilderCreateBasicType(
             DIB(cx),
-            name.as_ptr(),
+            name.as_c_char_ptr(),
+            name.len(),
             cx.size_of(t).bits(),
             encoding,
         )
-    }
+    };
+
+    DINodeCreationResult::new(ty_di_node, false)
 }
 
-fn foreign_type_metadata<'ll, 'tcx>(
+fn build_foreign_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     t: Ty<'tcx>,
-    unique_type_id: UniqueTypeId,
-) -> &'ll DIType {
-    let name = compute_debuginfo_type_name(cx.tcx, t, false);
-    create_struct_stub(
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    debug!("build_foreign_type_di_node: {:?}", t);
+
+    let &ty::Foreign(def_id) = unique_type_id.expect_ty().kind() else {
+        bug!(
+            "build_foreign_type_di_node() called with unexpected type: {:?}",
+            unique_type_id.expect_ty()
+        );
+    };
+
+    build_type_with_children(
         cx,
-        t,
-        &name,
-        unique_type_id,
-        NO_SCOPE_METADATA,
-        DIFlags::FlagZero,
+        type_map::stub(
+            cx,
+            Stub::Struct,
+            unique_type_id,
+            &compute_debuginfo_type_name(cx.tcx, t, false),
+            None,
+            cx.size_and_align_of(t),
+            Some(get_namespace_for_item(cx, def_id)),
+            DIFlags::FlagZero,
+        ),
+        |_, _| smallvec![],
+        NO_GENERICS,
     )
 }
 
-fn pointer_type_metadata<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    pointer_type: Ty<'tcx>,
-    pointee_type_metadata: &'ll DIType,
-) -> &'ll DIType {
-    let (pointer_size, pointer_align) = cx.size_and_align_of(pointer_type);
-    let name = CString::new(compute_debuginfo_type_name(cx.tcx, pointer_type, false)).unwrap();
-    unsafe {
-        llvm::LLVMRustDIBuilderCreatePointerType(
-            DIB(cx),
-            pointee_type_metadata,
-            pointer_size.bits(),
-            pointer_align.bits() as u32,
-            name.as_ptr(),
-        )
-    }
-}
-
-fn param_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType {
-    let name = CString::new(format!("{:?}", t)).unwrap();
-    unsafe {
-        llvm::LLVMRustDIBuilderCreateBasicType(
-            DIB(cx),
-            name.as_ptr(),
-            Size::ZERO.bits(),
-            DW_ATE_unsigned,
-        )
-    }
-}
-
-pub fn compile_unit_metadata<'ll>(
-    tcx: TyCtxt<'_>,
-    _codegen_unit_name: &str,
-    debug_context: &CrateDebugContext<'ll, '_>,
+pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>(
+    tcx: TyCtxt<'tcx>,
+    codegen_unit_name: &str,
+    debug_context: &CodegenUnitDebugContext<'ll, 'tcx>,
 ) -> &'ll DIDescriptor {
-    let name_in_debuginfo = match tcx.sess.local_crate_source_file {
-        Some(ref path) => path.clone(),
-        None => PathBuf::from(&*tcx.crate_name(LOCAL_CRATE).as_str()),
-    };
+    use rustc_session::RemapFileNameExt;
+    use rustc_session::config::RemapPathScopeComponents;
+    let mut name_in_debuginfo = tcx
+        .sess
+        .local_crate_source_file()
+        .map(|src| {
+            src.for_scope(&tcx.sess, RemapPathScopeComponents::DEBUGINFO)
+                .to_path_buf()
+        })
+        .unwrap_or_else(|| PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str()));
+
+    // To avoid breaking split DWARF, we need to ensure that each codegen unit
+    // has a unique `DW_AT_name`. This is because there's a remote chance that
+    // different codegen units for the same module will have entirely
+    // identical DWARF entries for the purpose of the DWO ID, which would
+    // violate Appendix F ("Split Dwarf Object Files") of the DWARF 5
+    // specification. LLVM uses the algorithm specified in section 7.32 "Type
+    // Signature Computation" to compute the DWO ID, which does not include
+    // any fields that would distinguish compilation units. So we must embed
+    // the codegen unit name into the `DW_AT_name`. (Issue #88521.)
+    //
+    // Additionally, the OSX linker has an idiosyncrasy where it will ignore
+    // some debuginfo if multiple object files with the same `DW_AT_name` are
+    // linked together.
+    //
+    // As a workaround for these two issues, we generate unique names for each
+    // object file. Those do not correspond to an actual source file but that
+    // is harmless.
+    name_in_debuginfo.push("@");
+    name_in_debuginfo.push(codegen_unit_name);
+
+    debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo);
 
     let rustc_producer = "rustc_codegen_nvvm".to_string();
 
@@ -992,206 +802,194 @@ pub fn compile_unit_metadata<'ll>(
     // gpu stuff is different
     let producer = format!("clang LLVM ({})", rustc_producer);
 
-    let name_in_debuginfo = name_in_debuginfo.to_string_lossy().into_owned();
-    let name_in_debuginfo = CString::new(name_in_debuginfo).unwrap();
-    let work_dir = CString::new(
-        &tcx.sess
-            .opts
-            .working_dir
-            .to_string_lossy(FileNameDisplayPreference::Remapped)[..],
-    )
-    .unwrap();
-    let producer = CString::new(producer).unwrap();
-    let flags = "\0";
-    let split_name = "\0";
-
-    assert!(tcx.sess.opts.debuginfo != DebugInfo::None);
+    let name_in_debuginfo = name_in_debuginfo.to_string_lossy();
+    let work_dir = tcx
+        .sess
+        .opts
+        .working_dir
+        .for_scope(tcx.sess, RemapPathScopeComponents::DEBUGINFO)
+        .to_string_lossy();
+    let output_filenames = tcx.output_filenames(());
+    let split_name = if tcx.sess.target_can_use_split_dwarf()
+        && let Some(f) = output_filenames.split_dwarf_path(
+            tcx.sess.split_debuginfo(),
+            tcx.sess.opts.unstable_opts.split_dwarf_kind,
+            Some(codegen_unit_name),
+        ) {
+        // We get a path relative to the working directory from split_dwarf_path
+        Some(tcx.sess.source_map().path_mapping().to_real_filename(f))
+    } else {
+        None
+    };
+    let split_name = split_name
+        .as_ref()
+        .map(|f| {
+            f.for_scope(tcx.sess, RemapPathScopeComponents::DEBUGINFO)
+                .to_string_lossy()
+        })
+        .unwrap_or_default();
+    let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo);
 
     unsafe {
-        let file_metadata = llvm::LLVMRustDIBuilderCreateFile(
+        let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile(
             debug_context.builder,
-            name_in_debuginfo.as_ptr(),
-            work_dir.as_ptr(),
+            name_in_debuginfo.as_c_char_ptr(),
+            name_in_debuginfo.len(),
+            work_dir.as_c_char_ptr(),
+            work_dir.len(),
+            llvm::ChecksumKind::None,
+            ptr::null(),
+            0,
+            ptr::null(),
+            0,
         );
 
-        llvm::LLVMRustDIBuilderCreateCompileUnit(
+        let unit_metadata = llvm::LLVMRustDIBuilderCreateCompileUnit(
             debug_context.builder,
-            DW_LANG_RUST,
-            file_metadata,
-            producer.as_ptr(),
+            dwarf_const::DW_LANG_Rust,
+            compile_unit_file,
+            producer.as_c_char_ptr(),
+            producer.len(),
             tcx.sess.opts.optimize != config::OptLevel::No,
-            flags.as_ptr() as *const _,
+            c"".as_ptr(),
             0,
-            split_name.as_ptr() as *const _,
-        )
-    }
-}
-
-struct MetadataCreationResult<'ll> {
-    metadata: &'ll DIType,
-    already_stored_in_typemap: bool,
-}
-
-impl<'ll> MetadataCreationResult<'ll> {
-    fn new(metadata: &'ll DIType, already_stored_in_typemap: bool) -> Self {
-        MetadataCreationResult {
-            metadata,
-            already_stored_in_typemap,
-        }
-    }
-}
+            // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead
+            // put the path supplied to `MCSplitDwarfFile` into the debug info of the final
+            // output(s).
+            split_name.as_c_char_ptr(),
+            split_name.len(),
+            kind,
+            0,
+            tcx.sess.opts.unstable_opts.split_dwarf_inlining,
+        );
 
-#[derive(Debug)]
-struct SourceInfo<'ll> {
-    file: &'ll DIFile,
-    line: u32,
+        return unit_metadata;
+    };
 }
 
-/// Description of a type member, which can either be a regular field (as in
-/// structs or tuples) or an enum variant.
-#[derive(Debug)]
-struct MemberDescription<'ll> {
-    name: String,
-    type_metadata: &'ll DIType,
+/// Creates a `DW_TAG_member` entry inside the DIE represented by the given `type_di_node`.
+fn build_field_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    owner: &'ll DIScope,
+    name: &str,
+    size_and_align: (Size, Align),
     offset: Size,
-    size: Size,
-    align: Align,
     flags: DIFlags,
-    discriminant: Option<u64>,
-    source_info: Option<SourceInfo<'ll>>,
-}
-
-impl<'ll> MemberDescription<'ll> {
-    fn into_metadata(
-        self,
-        cx: &CodegenCx<'ll, '_>,
-        composite_type_metadata: &'ll DIScope,
-    ) -> &'ll DIType {
-        let (file, line) = self
-            .source_info
-            .map(|info| (info.file, info.line))
-            .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER));
-
-        let name = CString::new(self.name).unwrap();
-        unsafe {
-            llvm::LLVMRustDIBuilderCreateVariantMemberType(
-                DIB(cx),
-                composite_type_metadata,
-                name.as_ptr(),
-                file,
-                line,
-                self.size.bits(),
-                self.align.bits() as u32,
-                self.offset.bits(),
-                self.discriminant.map(|v| cx.const_u64(v)),
-                self.flags,
-                self.type_metadata,
-            )
-        }
-    }
-}
-
-/// A factory for `MemberDescription`s. It produces a list of member descriptions
-/// for some record-like type. `MemberDescriptionFactory`s are used to defer the
-/// creation of type member descriptions in order to break cycles arising from
-/// recursive type definitions.
-enum MemberDescriptionFactory<'ll, 'tcx> {
-    StructMDF(StructMemberDescriptionFactory<'tcx>),
-    TupleMDF(TupleMemberDescriptionFactory<'tcx>),
-    EnumMDF(EnumMemberDescriptionFactory<'ll, 'tcx>),
-    UnionMDF(UnionMemberDescriptionFactory<'tcx>),
-    VariantMDF(VariantMemberDescriptionFactory<'tcx>),
-}
-
-use MemberDescriptionFactory::*;
-
-impl<'ll, 'tcx> MemberDescriptionFactory<'ll, 'tcx> {
-    fn create_member_descriptions(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<MemberDescription<'ll>> {
-        match *self {
-            StructMDF(ref this) => this.create_member_descriptions(cx),
-            TupleMDF(ref this) => this.create_member_descriptions(cx),
-            EnumMDF(ref this) => this.create_member_descriptions(cx),
-            UnionMDF(ref this) => this.create_member_descriptions(cx),
-            VariantMDF(ref this) => this.create_member_descriptions(cx),
-        }
+    type_di_node: &'ll DIType,
+    def_id: Option<DefId>,
+) -> &'ll DIType {
+    let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers
+    {
+        file_metadata_from_def_id(cx, def_id)
+    } else {
+        (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)
+    };
+    unsafe {
+        llvm::LLVMRustDIBuilderCreateMemberType(
+            DIB(cx),
+            owner,
+            name.as_c_char_ptr(),
+            name.len(),
+            file_metadata,
+            line_number,
+            size_and_align.0.bits(),
+            size_and_align.1.bits() as u32,
+            offset.bits(),
+            flags,
+            type_di_node,
+        )
     }
 }
 
-//=-----------------------------------------------------------------------------
-// Structs
-//=-----------------------------------------------------------------------------
-
-/// Creates `MemberDescription`s for the fields of a struct.
-struct StructMemberDescriptionFactory<'tcx> {
-    ty: Ty<'tcx>,
-    variant: &'tcx ty::VariantDef,
-    span: Span,
-}
-
-impl<'ll, 'tcx> StructMemberDescriptionFactory<'tcx> {
-    fn create_member_descriptions(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<MemberDescription<'ll>> {
-        let layout = cx.layout_of(self.ty);
-        self.variant
-            .fields
-            .iter()
-            .enumerate()
-            .map(|(i, f)| {
-                let name = if self.variant.ctor_kind == CtorKind::Fn {
-                    format!("__{}", i)
-                } else {
-                    f.ident.to_string()
-                };
-                let field = layout.field(cx, i);
-                MemberDescription {
-                    name,
-                    type_metadata: type_metadata(cx, field.ty, self.span),
-                    offset: layout.fields.offset(i),
-                    size: field.size,
-                    align: field.align.abi,
-                    flags: DIFlags::FlagZero,
-                    discriminant: None,
-                    source_info: None,
-                }
-            })
-            .collect()
+/// Returns the `DIFlags` corresponding to the visibility of the item identified by `did`.
+///
+/// `DIFlags::Flag{Public,Protected,Private}` correspond to `DW_AT_accessibility`
+/// (public/protected/private) aren't exactly right for Rust, but neither is `DW_AT_visibility`
+/// (local/exported/qualified), and there's no way to set `DW_AT_visibility` in LLVM's API.
+fn visibility_di_flags<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    did: DefId,
+    type_did: DefId,
+) -> DIFlags {
+    let parent_did = cx.tcx.parent(type_did);
+    let visibility = cx.tcx.visibility(did);
+    match visibility {
+        Visibility::Public => DIFlags::FlagPublic,
+        // Private fields have a restricted visibility of the module containing the type.
+        Visibility::Restricted(did) if did == parent_did => DIFlags::FlagPrivate,
+        // `pub(crate)`/`pub(super)` visibilities are any other restricted visibility.
+        Visibility::Restricted(..) => DIFlags::FlagProtected,
     }
 }
 
-fn prepare_struct_metadata<'ll, 'tcx>(
+/// Creates the debuginfo node for a Rust struct type. Maybe be a regular struct or a tuple-struct.
+fn build_struct_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
-    struct_type: Ty<'tcx>,
-    unique_type_id: UniqueTypeId,
-    span: Span,
-) -> RecursiveTypeDescription<'ll, 'tcx> {
-    let struct_name = compute_debuginfo_type_name(cx.tcx, struct_type, false);
-
-    let (struct_def_id, variant) = match struct_type.kind() {
-        ty::Adt(def, _) => (def.did, def.non_enum_variant()),
-        _ => bug!("prepare_struct_metadata on a non-ADT"),
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    let struct_type = unique_type_id.expect_ty();
+    let ty::Adt(adt_def, _) = struct_type.kind() else {
+        bug!(
+            "build_struct_type_di_node() called with non-struct-type: {:?}",
+            struct_type
+        );
+    };
+    assert!(adt_def.is_struct());
+    let containing_scope = get_namespace_for_item(cx, adt_def.did());
+    let struct_type_and_layout = cx.layout_of(struct_type);
+    let variant_def = adt_def.non_enum_variant();
+    let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+        Some(file_metadata_from_def_id(cx, Some(adt_def.did())))
+    } else {
+        None
     };
 
-    let containing_scope = get_namespace_for_item(cx, struct_def_id);
-
-    let struct_metadata_stub = create_struct_stub(
-        cx,
-        struct_type,
-        &struct_name,
-        unique_type_id,
-        Some(containing_scope),
-        DIFlags::FlagZero,
-    );
-
-    create_and_register_recursive_type_forward_declaration(
+    type_map::build_type_with_children(
         cx,
-        struct_type,
-        unique_type_id,
-        struct_metadata_stub,
-        struct_metadata_stub,
-        StructMDF(StructMemberDescriptionFactory {
-            ty: struct_type,
-            variant,
-            span,
-        }),
+        type_map::stub(
+            cx,
+            Stub::Struct,
+            unique_type_id,
+            &compute_debuginfo_type_name(cx.tcx, struct_type, false),
+            def_location,
+            size_and_align_of(struct_type_and_layout),
+            Some(containing_scope),
+            visibility_di_flags(cx, adt_def.did(), adt_def.did()),
+        ),
+        // Fields:
+        |cx, owner| {
+            variant_def
+                .fields
+                .iter()
+                .enumerate()
+                .map(|(i, f)| {
+                    let field_name = if variant_def.ctor_kind() == Some(CtorKind::Fn) {
+                        // This is a tuple struct
+                        tuple_field_name(i)
+                    } else {
+                        // This is struct with named fields
+                        Cow::Borrowed(f.name.as_str())
+                    };
+                    let field_layout = struct_type_and_layout.field(cx, i);
+                    let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+                        Some(f.did)
+                    } else {
+                        None
+                    };
+                    build_field_di_node(
+                        cx,
+                        owner,
+                        &field_name[..],
+                        (field_layout.size, field_layout.align.abi),
+                        struct_type_and_layout.fields.offset(i),
+                        visibility_di_flags(cx, f.did, adt_def.did()),
+                        type_di_node(cx, field_layout.ty),
+                        def_id,
+                    )
+                })
+                .collect()
+        },
+        |cx| build_generic_type_param_di_nodes(cx, struct_type),
     )
 }
 
@@ -1199,1050 +997,253 @@ fn prepare_struct_metadata<'ll, 'tcx>(
 // Tuples
 //=-----------------------------------------------------------------------------
 
-/// Returns names of captured upvars for closures and generators.
-///
-/// Here are some examples:
-///  - `name__field1__field2` when the upvar is captured by value.
-///  - `_ref__name__field` when the upvar is captured by reference.
-fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> Vec<String> {
-    let body = tcx.optimized_mir(def_id);
-
-    body.var_debug_info
-        .iter()
-        .filter_map(|var| {
-            let is_ref = match var.value {
-                mir::VarDebugInfoContents::Place(place) if place.local == mir::Local::new(1) => {
-                    // The projection is either `[.., Field, Deref]` or `[.., Field]`. It
-                    // implies whether the variable is captured by value or by reference.
-                    matches!(place.projection.last().unwrap(), mir::ProjectionElem::Deref)
-                }
-                _ => return None,
-            };
-            let prefix = if is_ref { "_ref__" } else { "" };
-            Some(prefix.to_owned() + &var.name.as_str())
-        })
-        .collect::<Vec<_>>()
-}
-
-/// Creates `MemberDescription`s for the fields of a tuple.
-struct TupleMemberDescriptionFactory<'tcx> {
-    ty: Ty<'tcx>,
-    component_types: Vec<Ty<'tcx>>,
-    span: Span,
-}
+/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or coroutine.
+/// For a coroutine, this will handle upvars shared by all states.
+fn build_upvar_field_di_nodes<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    closure_or_coroutine_ty: Ty<'tcx>,
+    closure_or_coroutine_di_node: &'ll DIType,
+) -> SmallVec<&'ll DIType> {
+    let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() {
+        ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()),
+        ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()),
+        ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()),
+        _ => {
+            bug!(
+                "build_upvar_field_di_nodes() called with non-closure-or-coroutine-type: {:?}",
+                closure_or_coroutine_ty
+            )
+        }
+    };
 
-impl<'tcx> TupleMemberDescriptionFactory<'tcx> {
-    fn create_member_descriptions<'ll>(
-        &self,
-        cx: &CodegenCx<'ll, 'tcx>,
-    ) -> Vec<MemberDescription<'ll>> {
-        let mut capture_names = match *self.ty.kind() {
-            ty::Generator(def_id, ..) | ty::Closure(def_id, ..) => {
-                Some(closure_saved_names_of_captured_variables(cx.tcx, def_id).into_iter())
-            }
-            _ => None,
-        };
-        let layout = cx.layout_of(self.ty);
-        self.component_types
+    assert!(
+        up_var_tys
             .iter()
-            .enumerate()
-            .map(|(i, &component_type)| {
-                let (size, align) = cx.size_and_align_of(component_type);
-                let name = if let Some(names) = capture_names.as_mut() {
-                    names.next().unwrap()
-                } else {
-                    format!("__{}", i)
-                };
-                MemberDescription {
-                    name,
-                    type_metadata: type_metadata(cx, component_type, self.span),
-                    offset: layout.fields.offset(i),
-                    size,
-                    align,
-                    flags: DIFlags::FlagZero,
-                    discriminant: None,
-                    source_info: None,
-                }
-            })
-            .collect()
-    }
-}
-
-fn prepare_tuple_metadata<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    tuple_type: Ty<'tcx>,
-    component_types: &[Ty<'tcx>],
-    unique_type_id: UniqueTypeId,
-    span: Span,
-    containing_scope: Option<&'ll DIScope>,
-) -> RecursiveTypeDescription<'ll, 'tcx> {
-    let tuple_name = compute_debuginfo_type_name(cx.tcx, tuple_type, false);
-
-    let struct_stub = create_struct_stub(
-        cx,
-        tuple_type,
-        &tuple_name[..],
-        unique_type_id,
-        containing_scope,
-        DIFlags::FlagZero,
+            .all(|t| t == cx.tcx.normalize_erasing_regions(cx.typing_env(), t))
     );
 
-    create_and_register_recursive_type_forward_declaration(
-        cx,
-        tuple_type,
-        unique_type_id,
-        struct_stub,
-        struct_stub,
-        TupleMDF(TupleMemberDescriptionFactory {
-            ty: tuple_type,
-            component_types: component_types.to_vec(),
-            span,
-        }),
-    )
-}
+    let capture_names = cx.tcx.closure_saved_names_of_captured_variables(def_id);
+    let layout = cx.layout_of(closure_or_coroutine_ty);
 
-//=-----------------------------------------------------------------------------
-// Unions
-//=-----------------------------------------------------------------------------
-
-struct UnionMemberDescriptionFactory<'tcx> {
-    layout: TyAndLayout<'tcx>,
-    variant: &'tcx ty::VariantDef,
-    span: Span,
-}
-
-impl<'tcx> UnionMemberDescriptionFactory<'tcx> {
-    fn create_member_descriptions<'ll>(
-        &self,
-        cx: &CodegenCx<'ll, 'tcx>,
-    ) -> Vec<MemberDescription<'ll>> {
-        self.variant
-            .fields
-            .iter()
-            .enumerate()
-            .map(|(i, f)| {
-                let field = self.layout.field(cx, i);
-                MemberDescription {
-                    name: f.ident.to_string(),
-                    type_metadata: type_metadata(cx, field.ty, self.span),
-                    offset: Size::ZERO,
-                    size: field.size,
-                    align: field.align.abi,
-                    flags: DIFlags::FlagZero,
-                    discriminant: None,
-                    source_info: None,
-                }
-            })
-            .collect()
-    }
+    up_var_tys
+        .into_iter()
+        .zip(capture_names.iter())
+        .enumerate()
+        .map(|(index, (up_var_ty, capture_name))| {
+            build_field_di_node(
+                cx,
+                closure_or_coroutine_di_node,
+                capture_name.as_str(),
+                cx.size_and_align_of(up_var_ty),
+                layout.fields.offset(index),
+                DIFlags::FlagZero,
+                type_di_node(cx, up_var_ty),
+                None,
+            )
+        })
+        .collect()
 }
 
-fn prepare_union_metadata<'ll, 'tcx>(
+/// Builds the DW_TAG_structure_type debuginfo node for a Rust tuple type.
+fn build_tuple_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
-    union_type: Ty<'tcx>,
-    unique_type_id: UniqueTypeId,
-    span: Span,
-) -> RecursiveTypeDescription<'ll, 'tcx> {
-    let union_name = compute_debuginfo_type_name(cx.tcx, union_type, false);
-
-    let (union_def_id, variant) = match union_type.kind() {
-        ty::Adt(def, _) => (def.did, def.non_enum_variant()),
-        _ => bug!("prepare_union_metadata on a non-ADT"),
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    let tuple_type = unique_type_id.expect_ty();
+    let &ty::Tuple(component_types) = tuple_type.kind() else {
+        bug!(
+            "build_tuple_type_di_node() called with non-tuple-type: {:?}",
+            tuple_type
+        )
     };
 
-    let containing_scope = get_namespace_for_item(cx, union_def_id);
+    let tuple_type_and_layout = cx.layout_of(tuple_type);
+    let type_name = compute_debuginfo_type_name(cx.tcx, tuple_type, false);
 
-    let union_metadata_stub = create_union_stub(
+    type_map::build_type_with_children(
         cx,
-        union_type,
-        &union_name,
-        unique_type_id,
-        containing_scope,
-    );
-
-    create_and_register_recursive_type_forward_declaration(
-        cx,
-        union_type,
-        unique_type_id,
-        union_metadata_stub,
-        union_metadata_stub,
-        UnionMDF(UnionMemberDescriptionFactory {
-            layout: cx.layout_of(union_type),
-            variant,
-            span,
-        }),
-    )
-}
-
-//=-----------------------------------------------------------------------------
-// Enums
-//=-----------------------------------------------------------------------------
-
-// FIXME(eddyb) maybe precompute this? Right now it's computed once
-// per generator monomorphization, but it doesn't depend on substs.
-fn generator_layout_and_saved_local_names(
-    tcx: TyCtxt<'_>,
-    def_id: DefId,
-) -> (
-    &'_ GeneratorLayout<'_>,
-    IndexVec<mir::GeneratorSavedLocal, Option<Symbol>>,
-) {
-    let body = tcx.optimized_mir(def_id);
-    let generator_layout = body.generator_layout().unwrap();
-    let mut generator_saved_local_names = IndexVec::from_elem(None, &generator_layout.field_tys);
-
-    let state_arg = mir::Local::new(1);
-    for var in &body.var_debug_info {
-        let place = if let mir::VarDebugInfoContents::Place(p) = var.value {
-            p
-        } else {
-            continue;
-        };
-        if place.local != state_arg {
-            continue;
-        }
-        if let [
-                // Deref of the `Pin<&mut Self>` state argument.
-                mir::ProjectionElem::Field(..),
-                mir::ProjectionElem::Deref,
-
-                // Field of a variant of the state.
-                mir::ProjectionElem::Downcast(_, variant),
-                mir::ProjectionElem::Field(field, _),
-            ] = place.projection[..] {
-            let name = &mut generator_saved_local_names[
-                generator_layout.variant_fields[variant][field]
-            ];
-            if name.is_none() {
-                name.replace(var.name);
-            }
-        }
-    }
-    (generator_layout, generator_saved_local_names)
-}
-
-/// Describes the members of an enum value; an enum is described as a union of
-/// structs in DWARF. This `MemberDescriptionFactory` provides the description for
-/// the members of this union; so for every variant of the given enum, this
-/// factory will produce one `MemberDescription` (all with no name and a fixed
-/// offset of zero bytes).
-struct EnumMemberDescriptionFactory<'ll, 'tcx> {
-    enum_type: Ty<'tcx>,
-    layout: TyAndLayout<'tcx>,
-    #[allow(dead_code)]
-    tag_type_metadata: Option<&'ll DIType>,
-    common_members: Vec<Option<&'ll DIType>>,
-    span: Span,
-}
-
-impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> {
-    fn create_member_descriptions(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<MemberDescription<'ll>> {
-        let generator_variant_info_data = match *self.enum_type.kind() {
-            ty::Generator(def_id, ..) => {
-                Some(generator_layout_and_saved_local_names(cx.tcx, def_id))
-            }
-            _ => None,
-        };
-
-        let variant_info_for = |index: VariantIdx| match *self.enum_type.kind() {
-            ty::Adt(adt, _) => VariantInfo::Adt(&adt.variants[index]),
-            ty::Generator(def_id, _, _) => {
-                let (generator_layout, generator_saved_local_names) =
-                    generator_variant_info_data.as_ref().unwrap();
-                VariantInfo::Generator {
-                    def_id,
-                    generator_layout: *generator_layout,
-                    generator_saved_local_names,
-                    variant_index: index,
-                }
-            }
-            _ => bug!(),
-        };
-
-        // This will always find the metadata in the type map.
-        let self_metadata = type_metadata(cx, self.enum_type, self.span);
-
-        match self.layout.variants {
-            Variants::Single { index } => {
-                if let ty::Adt(adt, _) = self.enum_type.kind() {
-                    if adt.variants.is_empty() {
-                        return vec![];
-                    }
-                }
-
-                let variant_info = variant_info_for(index);
-                let (variant_type_metadata, member_description_factory) =
-                    describe_enum_variant(cx, self.layout, variant_info, self_metadata, self.span);
-
-                let member_descriptions = member_description_factory.create_member_descriptions(cx);
-
-                set_members_of_composite_type(
-                    cx,
-                    self.enum_type,
-                    variant_type_metadata,
-                    member_descriptions,
-                    Some(&self.common_members),
-                );
-                vec![MemberDescription {
-                    name: variant_info.variant_name(),
-                    type_metadata: variant_type_metadata,
-                    offset: Size::ZERO,
-                    size: self.layout.size,
-                    align: self.layout.align.abi,
-                    flags: DIFlags::FlagZero,
-                    discriminant: None,
-                    source_info: variant_info.source_info(cx),
-                }]
-            }
-            Variants::Multiple {
-                tag_encoding: TagEncoding::Direct,
-                ref variants,
-                ..
-            } => variants
-                .iter_enumerated()
-                .map(|(i, _)| {
-                    let variant = self.layout.for_variant(cx, i);
-                    let variant_info = variant_info_for(i);
-                    let (variant_type_metadata, member_desc_factory) =
-                        describe_enum_variant(cx, variant, variant_info, self_metadata, self.span);
-
-                    let member_descriptions = member_desc_factory.create_member_descriptions(cx);
-
-                    set_members_of_composite_type(
+        type_map::stub(
+            cx,
+            Stub::Struct,
+            unique_type_id,
+            &type_name,
+            None,
+            size_and_align_of(tuple_type_and_layout),
+            NO_SCOPE_METADATA,
+            DIFlags::FlagZero,
+        ),
+        // Fields:
+        |cx, tuple_di_node| {
+            component_types
+                .into_iter()
+                .enumerate()
+                .map(|(index, component_type)| {
+                    build_field_di_node(
                         cx,
-                        self.enum_type,
-                        variant_type_metadata,
-                        member_descriptions,
-                        Some(&self.common_members),
-                    );
-
-                    MemberDescription {
-                        name: variant_info.variant_name(),
-                        type_metadata: variant_type_metadata,
-                        offset: Size::ZERO,
-                        size: self.layout.size,
-                        align: self.layout.align.abi,
-                        flags: DIFlags::FlagZero,
-                        discriminant: Some(
-                            self.layout
-                                .ty
-                                .discriminant_for_variant(cx.tcx, i)
-                                .unwrap()
-                                .val as u64,
-                        ),
-                        source_info: variant_info.source_info(cx),
-                    }
+                        tuple_di_node,
+                        &tuple_field_name(index),
+                        cx.size_and_align_of(component_type),
+                        tuple_type_and_layout.fields.offset(index),
+                        DIFlags::FlagZero,
+                        type_di_node(cx, component_type),
+                        None,
+                    )
                 })
-                .collect(),
-            Variants::Multiple {
-                tag_encoding:
-                    TagEncoding::Niche {
-                        ref niche_variants,
-                        niche_start,
-                        dataful_variant,
-                    },
-                ref tag,
-                ref variants,
-                ..
-            } => {
-                let calculate_niche_value = |i: VariantIdx| {
-                    if i == dataful_variant {
-                        None
-                    } else {
-                        let value = (i.as_u32() as u128)
-                            .wrapping_sub(niche_variants.start().as_u32() as u128)
-                            .wrapping_add(niche_start);
-                        let value = tag.value.size(cx).truncate(value);
-                        // NOTE(eddyb) do *NOT* remove this assert, until
-                        // we pass the full 128-bit value to LLVM, otherwise
-                        // truncation will be silent and remain undetected.
-                        assert_eq!(value as u64 as u128, value);
-                        Some(value as u64)
-                    }
-                };
-
-                variants
-                    .iter_enumerated()
-                    .map(|(i, _)| {
-                        let variant = self.layout.for_variant(cx, i);
-                        let variant_info = variant_info_for(i);
-                        let (variant_type_metadata, member_desc_factory) = describe_enum_variant(
-                            cx,
-                            variant,
-                            variant_info,
-                            self_metadata,
-                            self.span,
-                        );
-
-                        let member_descriptions =
-                            member_desc_factory.create_member_descriptions(cx);
-
-                        set_members_of_composite_type(
-                            cx,
-                            self.enum_type,
-                            variant_type_metadata,
-                            member_descriptions,
-                            Some(&self.common_members),
-                        );
-
-                        let niche_value = calculate_niche_value(i);
-
-                        MemberDescription {
-                            name: variant_info.variant_name(),
-                            type_metadata: variant_type_metadata,
-                            offset: Size::ZERO,
-                            size: self.layout.size,
-                            align: self.layout.align.abi,
-                            flags: DIFlags::FlagZero,
-                            discriminant: niche_value,
-                            source_info: variant_info.source_info(cx),
-                        }
-                    })
-                    .collect()
-            }
-        }
-    }
-}
-
-// Creates `MemberDescription`s for the fields of a single enum variant.
-struct VariantMemberDescriptionFactory<'tcx> {
-    /// Cloned from the `layout::Struct` describing the variant.
-    offsets: Vec<Size>,
-    args: Vec<(String, Ty<'tcx>)>,
-    span: Span,
-}
-
-impl<'tcx> VariantMemberDescriptionFactory<'tcx> {
-    fn create_member_descriptions<'ll>(
-        &self,
-        cx: &CodegenCx<'ll, 'tcx>,
-    ) -> Vec<MemberDescription<'ll>> {
-        self.args
-            .iter()
-            .enumerate()
-            .map(|(i, &(ref name, ty))| {
-                let (size, align) = cx.size_and_align_of(ty);
-                MemberDescription {
-                    name: name.to_string(),
-                    type_metadata: type_metadata(cx, ty, self.span),
-                    offset: self.offsets[i],
-                    size,
-                    align,
-                    flags: DIFlags::FlagZero,
-                    discriminant: None,
-                    source_info: None,
-                }
-            })
-            .collect()
-    }
-}
-
-#[derive(Copy, Clone)]
-enum VariantInfo<'a, 'tcx> {
-    Adt(&'tcx ty::VariantDef),
-    Generator {
-        def_id: DefId,
-        generator_layout: &'tcx GeneratorLayout<'tcx>,
-        generator_saved_local_names: &'a IndexVec<mir::GeneratorSavedLocal, Option<Symbol>>,
-        variant_index: VariantIdx,
-    },
+                .collect()
+        },
+        NO_GENERICS,
+    )
 }
 
-impl<'tcx> VariantInfo<'_, 'tcx> {
-    fn map_struct_name<R>(&self, f: impl FnOnce(&str) -> R) -> R {
-        match self {
-            VariantInfo::Adt(variant) => f(&variant.ident.as_str()),
-            VariantInfo::Generator { variant_index, .. } => {
-                f(&GeneratorSubsts::variant_name(*variant_index))
-            }
-        }
-    }
-
-    fn variant_name(&self) -> String {
-        match self {
-            VariantInfo::Adt(variant) => variant.ident.to_string(),
-            VariantInfo::Generator { variant_index, .. } => {
-                // Since GDB currently prints out the raw discriminant along
-                // with every variant, make each variant name be just the value
-                // of the discriminant. The struct name for the variant includes
-                // the actual variant description.
-                format!("{}", variant_index.as_usize())
-            }
-        }
-    }
-
-    fn field_name(&self, i: usize) -> String {
-        let field_name = match *self {
-            VariantInfo::Adt(variant) if variant.ctor_kind != CtorKind::Fn => {
-                Some(variant.fields[i].ident.name)
-            }
-            VariantInfo::Generator {
-                generator_layout,
-                generator_saved_local_names,
-                variant_index,
-                ..
-            } => {
-                generator_saved_local_names
-                    [generator_layout.variant_fields[variant_index][i.into()]]
-            }
-            _ => None,
-        };
-        field_name
-            .map(|name| name.to_string())
-            .unwrap_or_else(|| format!("__{}", i))
-    }
+/// Builds the debuginfo node for a closure environment.
+fn build_closure_env_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    let closure_env_type = unique_type_id.expect_ty();
+    let &(ty::Closure(def_id, _) | ty::CoroutineClosure(def_id, _)) = closure_env_type.kind()
+    else {
+        bug!(
+            "build_closure_env_di_node() called with non-closure-type: {:?}",
+            closure_env_type
+        )
+    };
+    let containing_scope = get_namespace_for_item(cx, def_id);
+    let type_name = compute_debuginfo_type_name(cx.tcx, closure_env_type, false);
 
-    fn source_info<'ll>(&self, cx: &CodegenCx<'ll, 'tcx>) -> Option<SourceInfo<'ll>> {
-        if let VariantInfo::Generator {
-            def_id,
-            variant_index,
-            ..
-        } = self
-        {
-            let span = cx
-                .tcx
-                .generator_layout(*def_id)
-                .unwrap()
-                .variant_source_info[*variant_index]
-                .span;
-            if !span.is_dummy() {
-                let loc = cx.lookup_debug_loc(span.lo());
-                return Some(SourceInfo {
-                    file: file_metadata(cx, &loc.file),
-                    line: loc.line,
-                });
-            }
-        }
+    let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+        Some(file_metadata_from_def_id(cx, Some(def_id)))
+    } else {
         None
-    }
-}
+    };
 
-/// Returns a tuple of (1) `type_metadata_stub` of the variant, (2) a
-/// `MemberDescriptionFactory` for producing the descriptions of the
-/// fields of the variant. This is a rudimentary version of a full
-/// `RecursiveTypeDescription`.
-fn describe_enum_variant<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    layout: layout::TyAndLayout<'tcx>,
-    variant: VariantInfo<'_, 'tcx>,
-    containing_scope: &'ll DIScope,
-    span: Span,
-) -> (&'ll DICompositeType, MemberDescriptionFactory<'ll, 'tcx>) {
-    let metadata_stub = variant.map_struct_name(|variant_name| {
-        let unique_type_id = debug_context(cx)
-            .type_map
-            .borrow_mut()
-            .get_unique_type_id_of_enum_variant(cx, layout.ty, variant_name);
-        create_struct_stub(
+    type_map::build_type_with_children(
+        cx,
+        type_map::stub(
             cx,
-            layout.ty,
-            variant_name,
+            Stub::Struct,
             unique_type_id,
+            &type_name,
+            def_location,
+            cx.size_and_align_of(closure_env_type),
             Some(containing_scope),
             DIFlags::FlagZero,
-        )
-    });
-
-    let offsets = (0..layout.fields.count())
-        .map(|i| layout.fields.offset(i))
-        .collect();
-    let args = (0..layout.fields.count())
-        .map(|i| (variant.field_name(i), layout.field(cx, i).ty))
-        .collect();
-
-    let member_description_factory = VariantMDF(VariantMemberDescriptionFactory {
-        offsets,
-        args,
-        span,
-    });
-
-    (metadata_stub, member_description_factory)
+        ),
+        // Fields:
+        |cx, owner| build_upvar_field_di_nodes(cx, closure_env_type, owner),
+        NO_GENERICS,
+    )
 }
 
-fn prepare_enum_metadata<'ll, 'tcx>(
+/// Build the debuginfo node for a Rust `union` type.
+fn build_union_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
-    enum_type: Ty<'tcx>,
-    enum_def_id: DefId,
-    unique_type_id: UniqueTypeId,
-    span: Span,
-    outer_field_tys: Vec<Ty<'tcx>>,
-) -> RecursiveTypeDescription<'ll, 'tcx> {
-    let tcx = cx.tcx;
-    let enum_name = compute_debuginfo_type_name(tcx, enum_type, false);
-
-    let containing_scope = get_namespace_for_item(cx, enum_def_id);
-    let file_metadata = unknown_file_metadata(cx);
-
-    let discriminant_type_metadata = |discr: Primitive| {
-        let enumerators_metadata: Vec<_> = match enum_type.kind() {
-            ty::Adt(def, _) => def
-                .discriminants(tcx)
-                .into_iter()
-                .zip(def.variants.iter())
-                .map(|((_, discr), v)| {
-                    let name = CString::new(&*v.ident.as_str()).unwrap();
-                    unsafe {
-                        Some(llvm::LLVMRustDIBuilderCreateEnumerator(
-                            DIB(cx),
-                            name.as_ptr(),
-                            discr.val as u64,
-                        ))
-                    }
-                })
-                .collect(),
-            ty::Generator(_, substs, _) => substs
-                .as_generator()
-                .variant_range(enum_def_id, tcx)
-                .map(|variant_index| {
-                    debug_assert_eq!(tcx.types.u32, substs.as_generator().discr_ty(tcx));
-                    let name =
-                        CString::new(&*GeneratorSubsts::variant_name(variant_index)).unwrap();
-                    unsafe {
-                        Some(llvm::LLVMRustDIBuilderCreateEnumerator(
-                            DIB(cx),
-                            name.as_ptr(),
-                            variant_index.as_u32().into(),
-                        ))
-                    }
-                })
-                .collect(),
-            _ => bug!(),
-        };
-        let disr_type_key = (enum_def_id, discr);
-        let cached_discriminant_type_metadata = debug_context(cx)
-            .created_enum_disr_types
-            .borrow()
-            .get(&disr_type_key)
-            .cloned();
-        match cached_discriminant_type_metadata {
-            Some(discriminant_type_metadata) => discriminant_type_metadata,
-            None => {
-                let (discriminant_size, discriminant_align) = (discr.size(cx), discr.align(cx));
-                let discriminant_base_type_metadata =
-                    type_metadata(cx, discr.to_ty(cx.tcx), DUMMY_SP);
-                let discriminant_name = get_enum_discriminant_name(cx, enum_def_id).as_str();
-
-                let name = CString::new(discriminant_name.as_bytes()).unwrap();
-                let discriminant_type_metadata = unsafe {
-                    llvm::LLVMRustDIBuilderCreateEnumerationType(
-                        DIB(cx),
-                        containing_scope,
-                        name.as_ptr(),
-                        file_metadata,
-                        UNKNOWN_LINE_NUMBER,
-                        discriminant_size.bits(),
-                        discriminant_align.abi.bits() as u32,
-                        create_DIArray(DIB(cx), &enumerators_metadata),
-                        discriminant_base_type_metadata,
-                    )
-                };
-
-                debug_context(cx)
-                    .created_enum_disr_types
-                    .borrow_mut()
-                    .insert(disr_type_key, discriminant_type_metadata);
-
-                discriminant_type_metadata
-            }
-        }
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    let union_type = unique_type_id.expect_ty();
+    let (union_def_id, variant_def) = match union_type.kind() {
+        ty::Adt(def, _) => (def.did(), def.non_enum_variant()),
+        _ => bug!("build_union_type_di_node on a non-ADT"),
     };
-
-    let layout = cx.layout_of(enum_type);
-
-    if let (
-        &Abi::Scalar(_),
-        &Variants::Multiple {
-            tag_encoding: TagEncoding::Direct,
-            ref tag,
-            ..
-        },
-    ) = (&layout.abi, &layout.variants)
-    {
-        return FinalMetadata(discriminant_type_metadata(tag.value));
-    }
-
-    let discriminator_name = CString::new(match enum_type.kind() {
-        ty::Generator(..) => "__state",
-        _ => "",
-    })
-    .unwrap();
-
-    let discriminator_metadata = match layout.variants {
-        // A single-variant enum has no discriminant.
-        Variants::Single { .. } => None,
-
-        Variants::Multiple {
-            tag_encoding: TagEncoding::Niche { .. },
-            ref tag,
-            tag_field,
-            ..
-        } => {
-            // Find the integer type of the correct size.
-            let size = tag.value.size(cx);
-            let align = tag.value.align(cx);
-
-            let tag_type = match tag.value {
-                Int(t, _) => t,
-                F32 => Integer::I32,
-                F64 => Integer::I64,
-                Pointer => cx.data_layout().ptr_sized_integer(),
-            }
-            .to_ty(cx.tcx, false);
-
-            let tag_metadata = basic_type_metadata(cx, tag_type);
-            unsafe {
-                Some(llvm::LLVMRustDIBuilderCreateMemberType(
-                    DIB(cx),
-                    containing_scope,
-                    discriminator_name.as_ptr(),
-                    file_metadata,
-                    UNKNOWN_LINE_NUMBER,
-                    size.bits(),
-                    align.abi.bits() as u32,
-                    layout.fields.offset(tag_field).bits(),
-                    DIFlags::FlagArtificial,
-                    tag_metadata,
-                ))
-            }
-        }
-
-        Variants::Multiple {
-            tag_encoding: TagEncoding::Direct,
-            ref tag,
-            tag_field,
-            ..
-        } => {
-            let discr_type = tag.value.to_ty(cx.tcx);
-            let (size, align) = cx.size_and_align_of(discr_type);
-
-            let discr_metadata = basic_type_metadata(cx, discr_type);
-            unsafe {
-                Some(llvm::LLVMRustDIBuilderCreateMemberType(
-                    DIB(cx),
-                    containing_scope,
-                    discriminator_name.as_ptr(),
-                    file_metadata,
-                    UNKNOWN_LINE_NUMBER,
-                    size.bits(),
-                    align.bits() as u32,
-                    layout.fields.offset(tag_field).bits(),
-                    DIFlags::FlagArtificial,
-                    discr_metadata,
-                ))
-            }
-        }
+    let containing_scope = get_namespace_for_item(cx, union_def_id);
+    let union_ty_and_layout = cx.layout_of(union_type);
+    let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false);
+    let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+        Some(file_metadata_from_def_id(cx, Some(union_def_id)))
+    } else {
+        None
     };
 
-    let outer_fields = match layout.variants {
-        Variants::Single { .. } => vec![],
-        Variants::Multiple { .. } => {
-            let tuple_mdf = TupleMemberDescriptionFactory {
-                ty: enum_type,
-                component_types: outer_field_tys,
-                span,
-            };
-            tuple_mdf
-                .create_member_descriptions(cx)
-                .into_iter()
-                .map(|desc| Some(desc.into_metadata(cx, containing_scope)))
+    type_map::build_type_with_children(
+        cx,
+        type_map::stub(
+            cx,
+            Stub::Union,
+            unique_type_id,
+            &type_name,
+            def_location,
+            size_and_align_of(union_ty_and_layout),
+            Some(containing_scope),
+            DIFlags::FlagZero,
+        ),
+        // Fields:
+        |cx, owner| {
+            variant_def
+                .fields
+                .iter()
+                .enumerate()
+                .map(|(i, f)| {
+                    let field_layout = union_ty_and_layout.field(cx, i);
+                    let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+                        Some(f.did)
+                    } else {
+                        None
+                    };
+                    build_field_di_node(
+                        cx,
+                        owner,
+                        f.name.as_str(),
+                        size_and_align_of(field_layout),
+                        Size::ZERO,
+                        DIFlags::FlagZero,
+                        type_di_node(cx, field_layout.ty),
+                        def_id,
+                    )
+                })
                 .collect()
-        }
-    };
-
-    let variant_part_unique_type_id_str = CString::new(
-        debug_context(cx)
-            .type_map
-            .borrow_mut()
-            .get_unique_type_id_str_of_enum_variant_part(unique_type_id),
+        },
+        // Generics:
+        |cx| build_generic_type_param_di_nodes(cx, union_type),
     )
-    .unwrap();
-    let empty_array = create_DIArray(DIB(cx), &[]);
-    let name = CString::new("").unwrap();
-    let variant_part = unsafe {
-        llvm::LLVMRustDIBuilderCreateVariantPart(
-            DIB(cx),
-            containing_scope,
-            name.as_ptr(),
-            file_metadata,
-            UNKNOWN_LINE_NUMBER,
-            layout.size.bits(),
-            layout.align.abi.bits() as u32,
-            DIFlags::FlagZero,
-            discriminator_metadata,
-            empty_array,
-            variant_part_unique_type_id_str.as_ptr(),
-        )
-    };
-
-    let struct_wrapper = {
-        // The variant part must be wrapped in a struct according to DWARF.
-        // All fields except the discriminant (including `outer_fields`)
-        // should be put into structures inside the variant part, which gives
-        // an equivalent layout but offers us much better integration with
-        // debuggers.
-        let type_array = create_DIArray(DIB(cx), &[Some(variant_part)]);
-
-        let type_map = debug_context(cx).type_map.borrow();
-        let unique_type_id_str =
-            CString::new(type_map.get_unique_type_id_as_string(unique_type_id)).unwrap();
-        let enum_name = CString::new(enum_name).unwrap();
-
-        unsafe {
-            llvm::LLVMRustDIBuilderCreateStructType(
-                DIB(cx),
-                Some(containing_scope),
-                enum_name.as_ptr(),
-                file_metadata,
-                UNKNOWN_LINE_NUMBER,
-                layout.size.bits(),
-                layout.align.abi.bits() as u32,
-                DIFlags::FlagZero,
-                None,
-                type_array,
-                0,
-                None,
-                unique_type_id_str.as_ptr(),
-            )
-        }
-    };
-
-    return create_and_register_recursive_type_forward_declaration(
-        cx,
-        enum_type,
-        unique_type_id,
-        struct_wrapper,
-        variant_part,
-        EnumMDF(EnumMemberDescriptionFactory {
-            enum_type,
-            layout,
-            tag_type_metadata: None,
-            common_members: outer_fields,
-            span,
-        }),
-    );
-
-    fn get_enum_discriminant_name(cx: &CodegenCx, def_id: DefId) -> Symbol {
-        cx.tcx.item_name(def_id)
-    }
-}
-
-/// Creates debug information for a composite type, that is, anything that
-/// results in a LLVM struct.
-///
-/// Examples of Rust types to use this are: structs, tuples, boxes, vecs, and enums.
-#[allow(clippy::too_many_arguments)]
-fn composite_type_metadata<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    composite_type: Ty<'tcx>,
-    composite_type_name: &str,
-    composite_type_unique_id: UniqueTypeId,
-    member_descriptions: Vec<MemberDescription<'ll>>,
-    containing_scope: Option<&'ll DIScope>,
-
-    // Ignore source location information as long as it
-    // can't be reconstructed for non-local crates.
-    _file_metadata: &'ll DIFile,
-    _definition_span: Span,
-) -> &'ll DICompositeType {
-    // Create the (empty) struct metadata node ...
-    let composite_type_metadata = create_struct_stub(
-        cx,
-        composite_type,
-        composite_type_name,
-        composite_type_unique_id,
-        containing_scope,
-        DIFlags::FlagZero,
-    );
-    // ... and immediately create and add the member descriptions.
-    set_members_of_composite_type(
-        cx,
-        composite_type,
-        composite_type_metadata,
-        member_descriptions,
-        None,
-    );
-
-    composite_type_metadata
-}
-
-fn set_members_of_composite_type<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    composite_type: Ty<'tcx>,
-    composite_type_metadata: &'ll DICompositeType,
-    member_descriptions: Vec<MemberDescription<'ll>>,
-    common_members: Option<&Vec<Option<&'ll DIType>>>,
-) {
-    // In some rare cases LLVM metadata uniquing would lead to an existing type
-    // description being used instead of a new one created in
-    // create_struct_stub. This would cause a hard to trace assertion in
-    // DICompositeType::SetTypeArray(). The following check makes sure that we
-    // get a better error message if this should happen again due to some
-    // regression.
-    {
-        let mut composite_types_completed =
-            debug_context(cx).composite_types_completed.borrow_mut();
-        if !composite_types_completed.insert(composite_type_metadata) {
-            bug!(
-                "debuginfo::set_members_of_composite_type() - \
-                  Already completed forward declaration re-encountered."
-            );
-        }
-    }
-
-    let mut member_metadata: Vec<_> = member_descriptions
-        .into_iter()
-        .map(|desc| Some(desc.into_metadata(cx, composite_type_metadata)))
-        .collect();
-    if let Some(other_members) = common_members {
-        member_metadata.extend(other_members.iter());
-    }
-
-    let type_params = compute_type_parameters(cx, composite_type);
-    unsafe {
-        let type_array = create_DIArray(DIB(cx), &member_metadata[..]);
-        llvm::LLVMRustDICompositeTypeReplaceArrays(
-            DIB(cx),
-            composite_type_metadata,
-            Some(type_array),
-            Some(type_params),
-        );
-    }
 }
 
 /// Computes the type parameters for a type, if any, for the given metadata.
-fn compute_type_parameters<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>) -> &'ll DIArray {
-    if let ty::Adt(def, substs) = *ty.kind() {
-        if substs.types().next().is_some() {
-            let generics = cx.tcx.generics_of(def.did);
+fn build_generic_type_param_di_nodes<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    ty: Ty<'tcx>,
+) -> SmallVec<&'ll DIType> {
+    if let ty::Adt(def, args) = *ty.kind() {
+        if args.types().next().is_some() {
+            let generics = cx.tcx.generics_of(def.did());
             let names = get_parameter_names(cx, generics);
-            let template_params: Vec<_> = substs
-                .into_iter()
-                .zip(names.into_iter())
+            let template_params: SmallVec<_> = iter::zip(args, names)
                 .filter_map(|(kind, name)| {
-                    if let GenericArgKind::Type(ty) = kind.unpack() {
-                        let actual_type =
-                            cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty);
-                        let actual_type_metadata =
-                            type_metadata(cx, actual_type, rustc_span::DUMMY_SP);
-                        let name = CString::new(&*name.as_str()).unwrap();
-                        Some(unsafe {
-                            Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter(
+                    kind.as_type().map(|ty| {
+                        let actual_type = cx.tcx.normalize_erasing_regions(cx.typing_env(), ty);
+                        let actual_type_di_node = type_di_node(cx, actual_type);
+                        let name = name.as_str();
+                        unsafe {
+                            llvm::LLVMRustDIBuilderCreateTemplateTypeParameter(
                                 DIB(cx),
                                 None,
-                                name.as_ptr(),
-                                actual_type_metadata,
-                            ))
-                        })
-                    } else {
-                        None
-                    }
+                                name.as_c_char_ptr(),
+                                name.len(),
+                                actual_type_di_node,
+                            )
+                        }
+                    })
                 })
                 .collect();
 
-            return create_DIArray(DIB(cx), &template_params[..]);
+            return template_params;
         }
     }
-    return create_DIArray(DIB(cx), &[]);
+
+    return smallvec![];
 
     fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec<Symbol> {
         let mut names = generics.parent.map_or_else(Vec::new, |def_id| {
             get_parameter_names(cx, cx.tcx.generics_of(def_id))
         });
-        names.extend(generics.params.iter().map(|param| param.name));
+        names.extend(generics.own_params.iter().map(|param| param.name));
         names
     }
 }
 
-/// A convenience wrapper around `LLVMRustDIBuilderCreateStructType()`. Does not do
-/// any caching, does not add any fields to the struct. This can be done later
-/// with `set_members_of_composite_type()`.
-fn create_struct_stub<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    struct_type: Ty<'tcx>,
-    struct_type_name: &str,
-    unique_type_id: UniqueTypeId,
-    containing_scope: Option<&'ll DIScope>,
-    flags: DIFlags,
-) -> &'ll DICompositeType {
-    let (struct_size, struct_align) = cx.size_and_align_of(struct_type);
-
-    let struct_type_name = CString::new(struct_type_name).unwrap();
-    let type_map = debug_context(cx).type_map.borrow();
-    let unique_type_id =
-        CString::new(type_map.get_unique_type_id_as_string(unique_type_id)).unwrap();
-
-    let metadata_stub = unsafe {
-        // `LLVMRustDIBuilderCreateStructType()` wants an empty array. A null
-        // pointer will lead to hard to trace and debug LLVM assertions
-        // later on in `llvm/lib/IR/Value.cpp`.
-        let empty_array = create_DIArray(DIB(cx), &[]);
-
-        llvm::LLVMRustDIBuilderCreateStructType(
-            DIB(cx),
-            containing_scope,
-            struct_type_name.as_ptr(),
-            unknown_file_metadata(cx),
-            UNKNOWN_LINE_NUMBER,
-            struct_size.bits(),
-            struct_align.bits() as u32,
-            flags,
-            None,
-            empty_array,
-            0,
-            None,
-            unique_type_id.as_ptr(),
-        )
-    };
-
-    metadata_stub
-}
-
-fn create_union_stub<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    union_type: Ty<'tcx>,
-    union_type_name: &str,
-    unique_type_id: UniqueTypeId,
-    containing_scope: &'ll DIScope,
-) -> &'ll DICompositeType {
-    let (union_size, union_align) = cx.size_and_align_of(union_type);
-
-    let union_type_name = CString::new(union_type_name).unwrap();
-    let type_map = debug_context(cx).type_map.borrow();
-    let unique_type_id =
-        CString::new(type_map.get_unique_type_id_as_string(unique_type_id)).unwrap();
-
-    let metadata_stub = unsafe {
-        // `LLVMRustDIBuilderCreateUnionType()` wants an empty array. A null
-        // pointer will lead to hard to trace and debug LLVM assertions
-        // later on in `llvm/lib/IR/Value.cpp`.
-        let empty_array = create_DIArray(DIB(cx), &[]);
-
-        llvm::LLVMRustDIBuilderCreateUnionType(
-            DIB(cx),
-            Some(containing_scope),
-            union_type_name.as_ptr(),
-            unknown_file_metadata(cx),
-            UNKNOWN_LINE_NUMBER,
-            union_size.bits(),
-            union_align.bits() as u32,
-            DIFlags::FlagZero,
-            Some(empty_array),
-            0, // RuntimeLang
-            unique_type_id.as_ptr(),
-        )
-    };
-
-    metadata_stub
-}
-
 /// Creates debug information for the given global variable.
 ///
 /// Adds the created metadata nodes directly to the crate's IR.
-pub(crate) fn create_global_var_metadata<'ll>(
-    cx: &CodegenCx<'ll, '_>,
-    def_id: DefId,
-    global: &'ll Value,
-) {
+pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, global: &'ll Value) {
     if cx.dbg_cx.is_none() {
         return;
     }
@@ -2257,28 +1258,29 @@ pub(crate) fn create_global_var_metadata<'ll>(
     // We may want to remove the namespace scope if we're in an extern block (see
     // https://github.com/rust-lang/rust/pull/46457#issuecomment-351750952).
     let var_scope = get_namespace_for_item(cx, def_id);
-    let span = tcx.def_span(def_id);
+    let (file_metadata, line_number) = file_metadata_from_def_id(cx, Some(def_id));
 
-    let (file_metadata, line_number) = if !span.is_dummy() {
-        let loc = cx.lookup_debug_loc(span.lo());
-        (file_metadata(cx, &loc.file), loc.line)
-    } else {
-        (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)
+    let is_local_to_unit = is_node_local_to_unit(cx, def_id);
+
+    let DefKind::Static { nested, .. } = cx.tcx.def_kind(def_id) else {
+        bug!()
     };
+    if nested {
+        return;
+    }
 
-    let is_local_to_unit = is_node_local_to_unit(cx, def_id);
-    let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all());
-    let type_metadata = type_metadata(cx, variable_type, span);
-    let var_name = CString::new(&*tcx.item_name(def_id).as_str()).unwrap();
+    let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, cx.typing_env());
+    let type_metadata = type_di_node(cx, variable_type);
+    let var_name = tcx.item_name(def_id);
+    let var_name = var_name.as_str();
     let linkage_name = mangled_name_of_instance(cx, Instance::mono(tcx, def_id)).name;
     // When empty, linkage_name field is omitted,
     // which is what we want for no_mangle statics
-    let linkage_name = CString::new(if var_name.to_str().unwrap() == linkage_name {
+    let linkage_name = if var_name == linkage_name {
         ""
     } else {
         linkage_name
-    })
-    .unwrap();
+    };
 
     let global_align = cx.align_of(variable_type);
 
@@ -2286,26 +1288,133 @@ pub(crate) fn create_global_var_metadata<'ll>(
         llvm::LLVMRustDIBuilderCreateStaticVariable(
             DIB(cx),
             Some(var_scope),
-            var_name.as_ptr(),
-            linkage_name.as_ptr(),
+            var_name.as_c_char_ptr(),
+            var_name.len(),
+            linkage_name.as_c_char_ptr(),
+            linkage_name.len(),
             file_metadata,
             line_number,
             type_metadata,
             is_local_to_unit,
             global,
             None,
-            global_align.bytes() as u32,
+            global_align.bits() as u32,
         );
     }
 }
 
+/// Generates LLVM debuginfo for a vtable.
+///
+/// The vtable type looks like a struct with a field for each function pointer and super-trait
+/// pointer it contains (plus the `size` and `align` fields).
+///
+/// Except for `size`, `align`, and `drop_in_place`, the field names don't try to mirror
+/// the name of the method they implement. This can be implemented in the future once there
+/// is a proper disambiguation scheme for dealing with methods from different traits that have
+/// the same name.
+fn build_vtable_type_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    ty: Ty<'tcx>,
+    poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
+) -> &'ll DIType {
+    let tcx = cx.tcx;
+
+    let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref {
+        let trait_ref = poly_trait_ref.with_self_ty(tcx, ty);
+        let trait_ref = tcx.erase_regions(trait_ref);
+
+        tcx.vtable_entries(trait_ref)
+    } else {
+        TyCtxt::COMMON_VTABLE_ENTRIES
+    };
+
+    // All function pointers are described as opaque pointers. This could be improved in the future
+    // by describing them as actual function pointers.
+    let void_pointer_ty = Ty::new_imm_ptr(tcx, tcx.types.unit);
+    let void_pointer_type_di_node = type_di_node(cx, void_pointer_ty);
+    let usize_di_node = type_di_node(cx, tcx.types.usize);
+    let (pointer_size, pointer_align) = cx.size_and_align_of(void_pointer_ty);
+    // If `usize` is not pointer-sized and -aligned then the size and alignment computations
+    // for the vtable as a whole would be wrong. Let's make sure this holds even on weird
+    // platforms.
+    assert_eq!(
+        cx.size_and_align_of(tcx.types.usize),
+        (pointer_size, pointer_align)
+    );
+
+    let vtable_type_name =
+        compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::Type);
+    let unique_type_id = UniqueTypeId::for_vtable_ty(tcx, ty, poly_trait_ref);
+    let size = pointer_size * vtable_entries.len() as u64;
+
+    // This gets mapped to a DW_AT_containing_type attribute which allows GDB to correlate
+    // the vtable to the type it is for.
+    let vtable_holder = type_di_node(cx, ty);
+
+    build_type_with_children(
+        cx,
+        type_map::stub(
+            cx,
+            Stub::VTableTy { vtable_holder },
+            unique_type_id,
+            &vtable_type_name,
+            None,
+            (size, pointer_align),
+            NO_SCOPE_METADATA,
+            DIFlags::FlagArtificial,
+        ),
+        |cx, vtable_type_di_node| {
+            vtable_entries
+                .iter()
+                .enumerate()
+                .filter_map(|(index, vtable_entry)| {
+                    let (field_name, field_type_di_node) = match vtable_entry {
+                        ty::VtblEntry::MetadataDropInPlace => {
+                            ("drop_in_place".to_string(), void_pointer_type_di_node)
+                        }
+                        ty::VtblEntry::Method(_) => {
+                            // Note: This code does not try to give a proper name to each method
+                            //       because their might be multiple methods with the same name
+                            //       (coming from different traits).
+                            (format!("__method{index}"), void_pointer_type_di_node)
+                        }
+                        ty::VtblEntry::TraitVPtr(_) => (
+                            format!("__super_trait_ptr{index}"),
+                            void_pointer_type_di_node,
+                        ),
+                        ty::VtblEntry::MetadataAlign => ("align".to_string(), usize_di_node),
+                        ty::VtblEntry::MetadataSize => ("size".to_string(), usize_di_node),
+                        ty::VtblEntry::Vacant => return None,
+                    };
+
+                    let field_offset = pointer_size * index as u64;
+
+                    Some(build_field_di_node(
+                        cx,
+                        vtable_type_di_node,
+                        &field_name,
+                        (pointer_size, pointer_align),
+                        field_offset,
+                        DIFlags::FlagZero,
+                        field_type_di_node,
+                        None,
+                    ))
+                })
+                .collect()
+        },
+        NO_GENERICS,
+    )
+    .di_node
+}
+
 /// Creates debug information for the given vtable, which is for the
 /// given type.
 ///
 /// Adds the created metadata nodes directly to the crate's IR.
-pub(crate) fn create_vtable_metadata<'ll, 'tcx>(
+pub(crate) fn create_vtable_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     ty: Ty<'tcx>,
+    poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
     vtable: &'ll Value,
 ) {
     if cx.dbg_cx.is_none() {
@@ -2317,49 +1426,27 @@ pub(crate) fn create_vtable_metadata<'ll, 'tcx>(
         return;
     }
 
-    let type_metadata = type_metadata(cx, ty, rustc_span::DUMMY_SP);
-
-    unsafe {
-        // LLVMRustDIBuilderCreateStructType() wants an empty array. A null
-        // pointer will lead to hard to trace and debug LLVM assertions
-        // later on in llvm/lib/IR/Value.cpp.
-        let empty_array = create_DIArray(DIB(cx), &[]);
+    // When full debuginfo is enabled, we want to try and prevent vtables from being
+    // merged. Otherwise debuggers will have a hard time mapping from dyn pointer
+    // to concrete type.
+    llvm::SetUnnamedAddress(vtable, llvm::UnnamedAddr::No);
 
-        let name = CString::new("vtable").unwrap();
-
-        // Create a new one each time.  We don't want metadata caching
-        // here, because each vtable will refer to a unique containing
-        // type.
-        let vtable_type = llvm::LLVMRustDIBuilderCreateStructType(
-            DIB(cx),
-            NO_SCOPE_METADATA,
-            name.as_ptr(),
-            unknown_file_metadata(cx),
-            UNKNOWN_LINE_NUMBER,
-            Size::ZERO.bits(),
-            cx.tcx.data_layout.pointer_align.abi.bits() as u32,
-            DIFlags::FlagArtificial,
-            None,
-            empty_array,
-            0,
-            Some(type_metadata),
-            name.as_ptr(),
-        );
+    let vtable_name =
+        compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable);
+    let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref);
+    let linkage_name = "";
 
+    unsafe {
         llvm::LLVMRustDIBuilderCreateStaticVariable(
             DIB(cx),
             NO_SCOPE_METADATA,
-            name.as_ptr(),
-            // LLVM 3.9
-            // doesn't accept
-            // null here, so
-            // pass the name
-            // as the linkage
-            // name.
-            name.as_ptr(),
+            vtable_name.as_c_char_ptr(),
+            vtable_name.len(),
+            linkage_name.as_c_char_ptr(),
+            linkage_name.len(),
             unknown_file_metadata(cx),
             UNKNOWN_LINE_NUMBER,
-            vtable_type,
+            vtable_type_di_node,
             true,
             vtable,
             None,
@@ -2377,3 +1464,31 @@ pub(crate) fn extend_scope_to_file<'ll>(
     let file_metadata = file_metadata(cx, file);
     unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlockFile(DIB(cx), scope_metadata, file_metadata) }
 }
+
+fn tuple_field_name(field_index: usize) -> Cow<'static, str> {
+    const TUPLE_FIELD_NAMES: [&'static str; 16] = [
+        "__0", "__1", "__2", "__3", "__4", "__5", "__6", "__7", "__8", "__9", "__10", "__11",
+        "__12", "__13", "__14", "__15",
+    ];
+    TUPLE_FIELD_NAMES
+        .get(field_index)
+        .map(|s| Cow::from(*s))
+        .unwrap_or_else(|| Cow::from(format!("__{field_index}")))
+}
+
+pub(crate) type DefinitionLocation<'ll> = (&'ll DIFile, c_uint);
+
+pub(crate) fn file_metadata_from_def_id<'ll>(
+    cx: &CodegenCx<'ll, '_>,
+    def_id: Option<DefId>,
+) -> DefinitionLocation<'ll> {
+    if let Some(def_id) = def_id
+        && let span = hygiene::walk_chain_collapsed(cx.tcx.def_span(def_id), DUMMY_SP)
+        && !span.is_dummy()
+    {
+        let loc = cx.lookup_debug_loc(span.lo());
+        (file_metadata(cx, &loc.file), loc.line)
+    } else {
+        (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)
+    }
+}
diff --git a/crates/rustc_codegen_nvvm/src/debug_info/metadata/enums.rs b/crates/rustc_codegen_nvvm/src/debug_info/metadata/enums.rs
new file mode 100644
index 00000000..f3d9051c
--- /dev/null
+++ b/crates/rustc_codegen_nvvm/src/debug_info/metadata/enums.rs
@@ -0,0 +1,878 @@
+use std::borrow::Cow;
+
+use libc::c_uint;
+use rustc_abi::{FieldIdx, Size, TagEncoding, VariantIdx, Variants};
+use rustc_codegen_ssa::debuginfo::{
+    tag_base_type, type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo,
+};
+use rustc_codegen_ssa::traits::{ConstCodegenMethods, MiscCodegenMethods};
+use rustc_hir::def::CtorKind;
+use rustc_index::IndexSlice;
+use rustc_middle::bug;
+use rustc_middle::mir::CoroutineLayout;
+use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
+use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef};
+use rustc_span::Symbol;
+use smallvec::smallvec;
+
+use super::type_map::{DINodeCreationResult, UniqueTypeId};
+use super::{SmallVec, size_and_align_of};
+use crate::common::AsCCharPtr;
+use crate::context::CodegenCx;
+use crate::debug_info::file_metadata_from_def_id;
+use crate::debug_info::metadata::type_map::{self, Stub, StubInfo};
+use crate::debug_info::metadata::{
+    NO_GENERICS, UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes,
+    file_metadata, type_di_node, unknown_file_metadata, visibility_di_flags,
+};
+use crate::debug_info::util::{DIB, create_DIArray, get_namespace_for_item};
+use crate::llvm::debuginfo::{DIFile, DIFlags, DIType};
+use crate::llvm::{self};
+
+/// Build the debuginfo node for an enum type. The listing below shows how such a
+/// type looks like at the LLVM IR/DWARF level. It is a `DW_TAG_structure_type`
+/// with a single `DW_TAG_variant_part` that in turn contains a `DW_TAG_variant`
+/// for each variant of the enum. The variant-part also contains a single member
+/// describing the discriminant, and a nested struct type for each of the variants.
+///
+/// ```txt
+///  ---> DW_TAG_structure_type              (top-level type for enum)
+///         DW_TAG_variant_part              (variant part)
+///           DW_AT_discr                    (reference to discriminant DW_TAG_member)
+///           DW_TAG_member                  (discriminant member)
+///           DW_TAG_variant                 (variant 1)
+///           DW_TAG_variant                 (variant 2)
+///           DW_TAG_variant                 (variant 3)
+///         DW_TAG_structure_type            (type of variant 1)
+///         DW_TAG_structure_type            (type of variant 2)
+///         DW_TAG_structure_type            (type of variant 3)
+/// ```
+pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    let enum_type = unique_type_id.expect_ty();
+    let &ty::Adt(enum_adt_def, _) = enum_type.kind() else {
+        bug!(
+            "build_enum_type_di_node() called with non-enum type: `{:?}`",
+            enum_type
+        )
+    };
+
+    let enum_type_and_layout = cx.layout_of(enum_type);
+
+    if wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout) {
+        return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout);
+    }
+
+    let containing_scope = get_namespace_for_item(cx, enum_adt_def.did());
+    let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false);
+
+    let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did());
+
+    let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+        Some(file_metadata_from_def_id(cx, Some(enum_adt_def.did())))
+    } else {
+        None
+    };
+
+    type_map::build_type_with_children(
+        cx,
+        type_map::stub(
+            cx,
+            Stub::Struct,
+            unique_type_id,
+            &enum_type_name,
+            def_location,
+            size_and_align_of(enum_type_and_layout),
+            Some(containing_scope),
+            visibility_flags,
+        ),
+        |cx, enum_type_di_node| {
+            // Build the struct type for each variant. These will be referenced by the
+            // DW_TAG_variant DIEs inside of the DW_TAG_variant_part DIE.
+            // We also called the names for the corresponding DW_TAG_variant DIEs here.
+            let variant_member_infos: SmallVec<_> = enum_adt_def
+                .variant_range()
+                .map(|variant_index| VariantMemberInfo {
+                    variant_index,
+                    variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()),
+                    variant_struct_type_di_node: build_enum_variant_struct_type_di_node(
+                        cx,
+                        enum_type_and_layout,
+                        enum_type_di_node,
+                        variant_index,
+                        enum_adt_def.variant(variant_index),
+                        enum_type_and_layout.for_variant(cx, variant_index),
+                        visibility_flags,
+                    ),
+                    source_info: None,
+                })
+                .collect();
+
+            let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+                Some(enum_adt_def.did())
+            } else {
+                None
+            };
+            smallvec![build_enum_variant_part_di_node(
+                cx,
+                enum_type_and_layout,
+                enum_type_di_node,
+                enum_adt_def_id,
+                &variant_member_infos[..],
+            )]
+        },
+        // We don't seem to be emitting generic args on the enum type, it seems. Rather
+        // they get attached to the struct type of each variant.
+        NO_GENERICS,
+    )
+}
+
+/// Build the debuginfo node for a coroutine environment. It looks the same as the debuginfo for
+/// an enum. See [build_enum_type_di_node] for more information.
+///
+/// ```txt
+///
+///  ---> DW_TAG_structure_type              (top-level type for the coroutine)
+///         DW_TAG_variant_part              (variant part)
+///           DW_AT_discr                    (reference to discriminant DW_TAG_member)
+///           DW_TAG_member                  (discriminant member)
+///           DW_TAG_variant                 (variant 1)
+///           DW_TAG_variant                 (variant 2)
+///           DW_TAG_variant                 (variant 3)
+///         DW_TAG_structure_type            (type of variant 1)
+///         DW_TAG_structure_type            (type of variant 2)
+///         DW_TAG_structure_type            (type of variant 3)
+///
+/// ```
+pub(super) fn build_coroutine_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    unique_type_id: UniqueTypeId<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    let coroutine_type = unique_type_id.expect_ty();
+    let &ty::Coroutine(coroutine_def_id, coroutine_args) = coroutine_type.kind() else {
+        bug!(
+            "build_coroutine_di_node() called with non-coroutine type: `{:?}`",
+            coroutine_type
+        )
+    };
+
+    let containing_scope = get_namespace_for_item(cx, coroutine_def_id);
+    let coroutine_type_and_layout = cx.layout_of(coroutine_type);
+
+    assert!(!wants_c_like_enum_debuginfo(
+        cx.tcx,
+        coroutine_type_and_layout
+    ));
+
+    let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false);
+
+    let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+        Some(file_metadata_from_def_id(cx, Some(coroutine_def_id)))
+    } else {
+        None
+    };
+
+    type_map::build_type_with_children(
+        cx,
+        type_map::stub(
+            cx,
+            Stub::Struct,
+            unique_type_id,
+            &coroutine_type_name,
+            def_location,
+            size_and_align_of(coroutine_type_and_layout),
+            Some(containing_scope),
+            DIFlags::FlagZero,
+        ),
+        |cx, coroutine_type_di_node| {
+            let coroutine_layout = cx
+                .tcx
+                .coroutine_layout(coroutine_def_id, coroutine_args.as_coroutine().kind_ty())
+                .unwrap();
+
+            let Variants::Multiple {
+                tag_encoding: TagEncoding::Direct,
+                ref variants,
+                ..
+            } = coroutine_type_and_layout.variants
+            else {
+                bug!(
+                    "Encountered coroutine with non-direct-tag layout: {:?}",
+                    coroutine_type_and_layout
+                )
+            };
+
+            let common_upvar_names = cx
+                .tcx
+                .closure_saved_names_of_captured_variables(coroutine_def_id);
+
+            // Build variant struct types
+            let variant_struct_type_di_nodes: SmallVec<_> = variants
+                .indices()
+                .map(|variant_index| {
+                    // FIXME: This is problematic because just a number is not a valid identifier.
+                    //        CoroutineArgs::variant_name(variant_index), would be consistent
+                    //        with enums?
+                    let variant_name = format!("{}", variant_index.as_usize()).into();
+
+                    let span = coroutine_layout.variant_source_info[variant_index].span;
+                    let source_info = if !span.is_dummy() {
+                        let loc = cx.lookup_debug_loc(span.lo());
+                        Some((file_metadata(cx, &loc.file), loc.line))
+                    } else {
+                        None
+                    };
+
+                    VariantMemberInfo {
+                        variant_index,
+                        variant_name,
+                        variant_struct_type_di_node: build_coroutine_variant_struct_type_di_node(
+                            cx,
+                            variant_index,
+                            coroutine_type_and_layout,
+                            coroutine_type_di_node,
+                            coroutine_layout,
+                            common_upvar_names,
+                        ),
+                        source_info,
+                    }
+                })
+                .collect();
+
+            let coroutine_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+                Some(coroutine_def_id)
+            } else {
+                None
+            };
+            smallvec![build_enum_variant_part_di_node(
+                cx,
+                coroutine_type_and_layout,
+                coroutine_type_di_node,
+                coroutine_def_id,
+                &variant_struct_type_di_nodes[..],
+            )]
+        },
+        // We don't seem to be emitting generic args on the coroutine type, it seems. Rather
+        // they get attached to the struct type of each variant.
+        NO_GENERICS,
+    )
+}
+
+/// Builds the DW_TAG_variant_part of an enum or coroutine debuginfo node:
+///
+/// ```txt
+///       DW_TAG_structure_type              (top-level type for enum)
+/// --->    DW_TAG_variant_part              (variant part)
+///           DW_AT_discr                    (reference to discriminant DW_TAG_member)
+///           DW_TAG_member                  (discriminant member)
+///           DW_TAG_variant                 (variant 1)
+///           DW_TAG_variant                 (variant 2)
+///           DW_TAG_variant                 (variant 3)
+///         DW_TAG_structure_type            (type of variant 1)
+///         DW_TAG_structure_type            (type of variant 2)
+///         DW_TAG_structure_type            (type of variant 3)
+/// ```
+fn build_enum_variant_part_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    enum_type_and_layout: TyAndLayout<'tcx>,
+    enum_type_di_node: &'ll DIType,
+    enum_type_def_id: Option<rustc_span::def_id::DefId>,
+    variant_member_infos: &[VariantMemberInfo<'_, 'll>],
+) -> &'ll DIType {
+    let tag_member_di_node =
+        build_discr_member_di_node(cx, enum_type_and_layout, enum_type_di_node);
+
+    let variant_part_unique_type_id =
+        UniqueTypeId::for_enum_variant_part(cx.tcx, enum_type_and_layout.ty);
+
+    let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers
+    {
+        file_metadata_from_def_id(cx, enum_type_def_id)
+    } else {
+        (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)
+    };
+
+    let stub = StubInfo::new(
+        cx,
+        variant_part_unique_type_id,
+        |cx, variant_part_unique_type_id_str| unsafe {
+            let variant_part_name = "";
+            llvm::LLVMRustDIBuilderCreateVariantPart(
+                DIB(cx),
+                enum_type_di_node,
+                variant_part_name.as_c_char_ptr(),
+                variant_part_name.len(),
+                file_metadata,
+                line_number,
+                enum_type_and_layout.size.bits(),
+                enum_type_and_layout.align.abi.bits() as u32,
+                DIFlags::FlagZero,
+                tag_member_di_node,
+                create_DIArray(DIB(cx), &[]),
+                variant_part_unique_type_id_str.as_c_char_ptr(),
+                variant_part_unique_type_id_str.len(),
+            )
+        },
+    );
+
+    type_map::build_type_with_children(
+        cx,
+        stub,
+        |cx, variant_part_di_node| {
+            variant_member_infos
+                .iter()
+                .map(|variant_member_info| {
+                    build_enum_variant_member_di_node(
+                        cx,
+                        enum_type_and_layout,
+                        variant_part_di_node,
+                        variant_member_info,
+                    )
+                })
+                .collect()
+        },
+        NO_GENERICS,
+    )
+    .di_node
+}
+
+/// Builds the DW_TAG_member describing where we can find the tag of an enum.
+/// Returns `None` if the enum does not have a tag.
+///
+/// ```txt
+///
+///       DW_TAG_structure_type              (top-level type for enum)
+///         DW_TAG_variant_part              (variant part)
+///           DW_AT_discr                    (reference to discriminant DW_TAG_member)
+/// --->      DW_TAG_member                  (discriminant member)
+///           DW_TAG_variant                 (variant 1)
+///           DW_TAG_variant                 (variant 2)
+///           DW_TAG_variant                 (variant 3)
+///         DW_TAG_structure_type            (type of variant 1)
+///         DW_TAG_structure_type            (type of variant 2)
+///         DW_TAG_structure_type            (type of variant 3)
+///
+/// ```
+fn build_discr_member_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    enum_or_coroutine_type_and_layout: TyAndLayout<'tcx>,
+    enum_or_coroutine_type_di_node: &'ll DIType,
+) -> Option<&'ll DIType> {
+    let tag_name = match enum_or_coroutine_type_and_layout.ty.kind() {
+        ty::Coroutine(..) => "__state",
+        _ => "",
+    };
+
+    // NOTE: This is actually wrong. This will become a member of
+    //       of the DW_TAG_variant_part. But, due to LLVM's API, that
+    //       can only be constructed with this DW_TAG_member already in created.
+    //       In LLVM IR the wrong scope will be listed but when DWARF is
+    //       generated from it, the DW_TAG_member will be a child the
+    //       DW_TAG_variant_part.
+    let containing_scope = enum_or_coroutine_type_di_node;
+
+    match enum_or_coroutine_type_and_layout.layout.variants() {
+        // A single-variant or no-variant enum has no discriminant.
+        &Variants::Single { .. } | &Variants::Empty => None,
+
+        &Variants::Multiple { tag_field, .. } => {
+            let tag_base_type = tag_base_type(cx.tcx, enum_or_coroutine_type_and_layout);
+            let (size, align) = cx.size_and_align_of(tag_base_type);
+
+            unsafe {
+                Some(llvm::LLVMRustDIBuilderCreateMemberType(
+                    DIB(cx),
+                    containing_scope,
+                    tag_name.as_c_char_ptr(),
+                    tag_name.len(),
+                    unknown_file_metadata(cx),
+                    UNKNOWN_LINE_NUMBER,
+                    size.bits(),
+                    align.bits() as u32,
+                    enum_or_coroutine_type_and_layout
+                        .fields
+                        .offset(tag_field)
+                        .bits(),
+                    DIFlags::FlagArtificial,
+                    type_di_node(cx, tag_base_type),
+                ))
+            }
+        }
+    }
+}
+
+/// Build the debuginfo node for `DW_TAG_variant`:
+///
+/// ```txt
+///       DW_TAG_structure_type              (top-level type for enum)
+///         DW_TAG_variant_part              (variant part)
+///           DW_AT_discr                    (reference to discriminant DW_TAG_member)
+///           DW_TAG_member                  (discriminant member)
+///  --->     DW_TAG_variant                 (variant 1)
+///  --->     DW_TAG_variant                 (variant 2)
+///  --->     DW_TAG_variant                 (variant 3)
+///         DW_TAG_structure_type            (type of variant 1)
+///         DW_TAG_structure_type            (type of variant 2)
+///         DW_TAG_structure_type            (type of variant 3)
+/// ```
+///
+/// This node looks like:
+///
+/// ```txt
+/// DW_TAG_variant
+///   DW_AT_discr_value           0
+///   DW_TAG_member
+///     DW_AT_name                  None
+///     DW_AT_type                  <0x000002a1>
+///     DW_AT_alignment             0x00000002
+///     DW_AT_data_member_location  0
+/// ```
+///
+/// The DW_AT_discr_value is optional, and is omitted if
+///   - This is the only variant of a univariant enum (i.e. their is no discriminant)
+///   - This is the "untagged" variant of a niche-layout enum
+///     (where only the other variants are identified by a single value)
+///
+/// There is only ever a single member, the type of which is a struct that describes the
+/// fields of the variant (excluding the discriminant). The name of the member is the name
+/// of the variant as given in the source code. The DW_AT_data_member_location is always
+/// zero.
+///
+/// Note that the LLVM DIBuilder API is a bit unintuitive here. The DW_TAG_variant subtree
+/// (including the DW_TAG_member) is built by a single call to
+/// `LLVMRustDIBuilderCreateVariantMemberType()`.
+fn build_enum_variant_member_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    enum_type_and_layout: TyAndLayout<'tcx>,
+    variant_part_di_node: &'ll DIType,
+    variant_member_info: &VariantMemberInfo<'_, 'll>,
+) -> &'ll DIType {
+    let variant_index = variant_member_info.variant_index;
+    let discr_value = compute_discriminant_value(cx, enum_type_and_layout, variant_index);
+
+    let (file_di_node, line_number) = variant_member_info
+        .source_info
+        .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER));
+
+    let discr = discr_value.opt_single_val().map(|value| {
+        let tag_base_type = tag_base_type(cx.tcx, enum_type_and_layout);
+        let size = cx.size_of(tag_base_type);
+        cx.const_uint_big(cx.type_ix(size.bits()), value)
+    });
+
+    unsafe {
+        llvm::LLVMRustDIBuilderCreateVariantMemberType(
+            DIB(cx),
+            variant_part_di_node,
+            variant_member_info.variant_name.as_c_char_ptr(),
+            variant_member_info.variant_name.len(),
+            file_di_node,
+            line_number,
+            enum_type_and_layout.size.bits(),
+            enum_type_and_layout.align.abi.bits() as u32,
+            Size::ZERO.bits(),
+            discr,
+            DIFlags::FlagZero,
+            variant_member_info.variant_struct_type_di_node,
+        )
+    }
+}
+
+/// Information needed for building a `DW_TAG_variant`:
+///
+/// ```txt
+///       DW_TAG_structure_type              (top-level type for enum)
+///         DW_TAG_variant_part              (variant part)
+///           DW_AT_discr                    (reference to discriminant DW_TAG_member)
+///           DW_TAG_member                  (discriminant member)
+///  --->     DW_TAG_variant                 (variant 1)
+///  --->     DW_TAG_variant                 (variant 2)
+///  --->     DW_TAG_variant                 (variant 3)
+///         DW_TAG_structure_type            (type of variant 1)
+///         DW_TAG_structure_type            (type of variant 2)
+///         DW_TAG_structure_type            (type of variant 3)
+/// ```
+struct VariantMemberInfo<'a, 'll> {
+    variant_index: VariantIdx,
+    variant_name: Cow<'a, str>,
+    variant_struct_type_di_node: &'ll DIType,
+    source_info: Option<(&'ll DIFile, c_uint)>,
+}
+
+/// Build the debuginfo node for a C-style enum, i.e. an enum the variants of which have no fields.
+///
+/// The resulting debuginfo will be a DW_TAG_enumeration_type.
+fn build_c_style_enum_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    enum_adt_def: AdtDef<'tcx>,
+    enum_type_and_layout: TyAndLayout<'tcx>,
+) -> DINodeCreationResult<'ll> {
+    let containing_scope = get_namespace_for_item(cx, enum_adt_def.did());
+    let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+        Some(enum_adt_def.did())
+    } else {
+        None
+    };
+    DINodeCreationResult {
+        di_node: build_enumeration_type_di_node(
+            cx,
+            &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false),
+            tag_base_type(cx.tcx, enum_type_and_layout),
+            enum_adt_def
+                .discriminants(cx.tcx)
+                .map(|(variant_index, discr)| {
+                    let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str());
+                    (name, discr.val)
+                }),
+            enum_adt_def_id,
+            containing_scope,
+        ),
+        already_stored_in_typemap: false,
+    }
+}
+
+/// Build a DW_TAG_enumeration_type debuginfo node, with the given base type and variants.
+/// This is a helper function and does not register anything in the type map by itself.
+///
+/// `variants` is an iterator of (discr-value, variant-name).
+///
+/// NVVM: Discrimant values are mapped from u128 to i64 to conform with LLVM 7's API.
+fn build_enumeration_type_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    type_name: &str,
+    base_type: Ty<'tcx>,
+    enumerators: impl Iterator<Item = (Cow<'tcx, str>, u128)>,
+    def_id: Option<rustc_span::def_id::DefId>,
+    containing_scope: &'ll DIType,
+) -> &'ll DIType {
+    let is_unsigned = match base_type.kind() {
+        ty::Int(_) => false,
+        ty::Uint(_) => true,
+        _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."),
+    };
+    let (size, align) = cx.size_and_align_of(base_type);
+
+    let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = enumerators
+        .map(|(name, value)| unsafe {
+            // FIXME: pass all 128 bits with newer LLVM API.
+            Some(llvm::LLVMRustDIBuilderCreateEnumerator(
+                DIB(cx),
+                name.as_c_char_ptr(),
+                name.len(),
+                value as i64,
+                is_unsigned,
+            ))
+        })
+        .collect();
+
+    let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers
+    {
+        file_metadata_from_def_id(cx, def_id)
+    } else {
+        (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)
+    };
+
+    unsafe {
+        llvm::LLVMRustDIBuilderCreateEnumerationType(
+            DIB(cx),
+            containing_scope,
+            type_name.as_c_char_ptr(),
+            type_name.len(),
+            file_metadata,
+            line_number,
+            size.bits(),
+            align.bits() as u32,
+            create_DIArray(DIB(cx), &enumerator_di_nodes[..]),
+            type_di_node(cx, base_type),
+        )
+    }
+}
+
+/// Build the debuginfo node for the struct type describing a single variant of an enum.
+///
+/// ```txt
+///       DW_TAG_structure_type              (top-level type for enum)
+///         DW_TAG_variant_part              (variant part)
+///           DW_AT_discr                    (reference to discriminant DW_TAG_member)
+///           DW_TAG_member                  (discriminant member)
+///           DW_TAG_variant                 (variant 1)
+///           DW_TAG_variant                 (variant 2)
+///           DW_TAG_variant                 (variant 3)
+///  --->   DW_TAG_structure_type            (type of variant 1)
+///  --->   DW_TAG_structure_type            (type of variant 2)
+///  --->   DW_TAG_structure_type            (type of variant 3)
+/// ```
+///
+/// The node looks like:
+///
+/// ```txt
+/// DW_TAG_structure_type
+///   DW_AT_name                  <name-of-variant>
+///   DW_AT_byte_size             0x00000010
+///   DW_AT_alignment             0x00000008
+///   DW_TAG_member
+///     DW_AT_name                  <name-of-field-0>
+///     DW_AT_type                  <0x0000018e>
+///     DW_AT_alignment             0x00000004
+///     DW_AT_data_member_location  4
+///   DW_TAG_member
+///     DW_AT_name                  <name-of-field-1>
+///     DW_AT_type                  <0x00000195>
+///     DW_AT_alignment             0x00000008
+///     DW_AT_data_member_location  8
+///   ...
+/// ```
+///
+/// The type of a variant is always a struct type with the name of the variant
+/// and a DW_TAG_member for each field (but not the discriminant).
+fn build_enum_variant_struct_type_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    enum_type_and_layout: TyAndLayout<'tcx>,
+    enum_type_di_node: &'ll DIType,
+    variant_index: VariantIdx,
+    variant_def: &VariantDef,
+    variant_layout: TyAndLayout<'tcx>,
+    di_flags: DIFlags,
+) -> &'ll DIType {
+    assert_eq!(variant_layout.ty, enum_type_and_layout.ty);
+
+    let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
+        Some(file_metadata_from_def_id(cx, Some(variant_def.def_id)))
+    } else {
+        None
+    };
+
+    type_map::build_type_with_children(
+        cx,
+        type_map::stub(
+            cx,
+            Stub::Struct,
+            UniqueTypeId::for_enum_variant_struct_type(
+                cx.tcx,
+                enum_type_and_layout.ty,
+                variant_index,
+            ),
+            variant_def.name.as_str(),
+            def_location,
+            // NOTE: We use size and align of enum_type, not from variant_layout:
+            size_and_align_of(enum_type_and_layout),
+            Some(enum_type_di_node),
+            di_flags,
+        ),
+        |cx, struct_type_di_node| {
+            (0..variant_layout.fields.count())
+                .map(|field_index| {
+                    let field_name = if variant_def.ctor_kind() != Some(CtorKind::Fn) {
+                        // Fields have names
+                        let field = &variant_def.fields[FieldIdx::from_usize(field_index)];
+                        Cow::from(field.name.as_str())
+                    } else {
+                        // Tuple-like
+                        super::tuple_field_name(field_index)
+                    };
+
+                    let field_layout = variant_layout.field(cx, field_index);
+
+                    build_field_di_node(
+                        cx,
+                        struct_type_di_node,
+                        &field_name,
+                        (field_layout.size, field_layout.align.abi),
+                        variant_layout.fields.offset(field_index),
+                        di_flags,
+                        type_di_node(cx, field_layout.ty),
+                        None,
+                    )
+                })
+                .collect::<SmallVec<_>>()
+        },
+        |cx| build_generic_type_param_di_nodes(cx, enum_type_and_layout.ty),
+    )
+    .di_node
+}
+
+/// Build the struct type for describing a single coroutine state.
+/// See [build_coroutine_variant_struct_type_di_node].
+///
+/// ```txt
+///
+///       DW_TAG_structure_type              (top-level type for enum)
+///         DW_TAG_variant_part              (variant part)
+///           DW_AT_discr                    (reference to discriminant DW_TAG_member)
+///           DW_TAG_member                  (discriminant member)
+///           DW_TAG_variant                 (variant 1)
+///           DW_TAG_variant                 (variant 2)
+///           DW_TAG_variant                 (variant 3)
+///  --->   DW_TAG_structure_type            (type of variant 1)
+///  --->   DW_TAG_structure_type            (type of variant 2)
+///  --->   DW_TAG_structure_type            (type of variant 3)
+///
+/// ```
+fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    variant_index: VariantIdx,
+    coroutine_type_and_layout: TyAndLayout<'tcx>,
+    coroutine_type_di_node: &'ll DIType,
+    coroutine_layout: &CoroutineLayout<'tcx>,
+    common_upvar_names: &IndexSlice<FieldIdx, Symbol>,
+) -> &'ll DIType {
+    let variant_name = CoroutineArgs::variant_name(variant_index);
+    let unique_type_id = UniqueTypeId::for_enum_variant_struct_type(
+        cx.tcx,
+        coroutine_type_and_layout.ty,
+        variant_index,
+    );
+
+    let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index);
+
+    let coroutine_args = match coroutine_type_and_layout.ty.kind() {
+        ty::Coroutine(_, args) => args.as_coroutine(),
+        _ => unreachable!(),
+    };
+
+    type_map::build_type_with_children(
+        cx,
+        type_map::stub(
+            cx,
+            Stub::Struct,
+            unique_type_id,
+            &variant_name,
+            None,
+            size_and_align_of(coroutine_type_and_layout),
+            Some(coroutine_type_di_node),
+            DIFlags::FlagZero,
+        ),
+        |cx, variant_struct_type_di_node| {
+            // Fields that just belong to this variant/state
+            let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count())
+                .map(|field_index| {
+                    let coroutine_saved_local = coroutine_layout.variant_fields[variant_index]
+                        [FieldIdx::from_usize(field_index)];
+                    let field_name_maybe = coroutine_layout.field_names[coroutine_saved_local];
+                    let field_name = field_name_maybe
+                        .as_ref()
+                        .map(|s| Cow::from(s.as_str()))
+                        .unwrap_or_else(|| super::tuple_field_name(field_index));
+
+                    let field_type = variant_layout.field(cx, field_index).ty;
+
+                    build_field_di_node(
+                        cx,
+                        variant_struct_type_di_node,
+                        &field_name,
+                        cx.size_and_align_of(field_type),
+                        variant_layout.fields.offset(field_index),
+                        DIFlags::FlagZero,
+                        type_di_node(cx, field_type),
+                        None,
+                    )
+                })
+                .collect();
+
+            // Fields that are common to all states
+            let common_fields: SmallVec<_> = coroutine_args
+                .prefix_tys()
+                .iter()
+                .zip(common_upvar_names)
+                .enumerate()
+                .map(|(index, (upvar_ty, upvar_name))| {
+                    build_field_di_node(
+                        cx,
+                        variant_struct_type_di_node,
+                        upvar_name.as_str(),
+                        cx.size_and_align_of(upvar_ty),
+                        coroutine_type_and_layout.fields.offset(index),
+                        DIFlags::FlagZero,
+                        type_di_node(cx, upvar_ty),
+                        None,
+                    )
+                })
+                .collect();
+
+            state_specific_fields
+                .into_iter()
+                .chain(common_fields)
+                .collect()
+        },
+        |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty),
+    )
+    .di_node
+}
+
+#[derive(Copy, Clone)]
+enum DiscrResult {
+    NoDiscriminant,
+    Value(u128),
+    #[allow(dead_code)]
+    Range(u128, u128),
+}
+
+impl DiscrResult {
+    fn opt_single_val(&self) -> Option<u128> {
+        if let Self::Value(d) = *self {
+            Some(d)
+        } else {
+            None
+        }
+    }
+}
+
+/// Returns the discriminant value corresponding to the variant index.
+///
+/// Will return `None` if there is less than two variants (because then the enum won't have)
+/// a tag, and if this is the untagged variant of a niche-layout enum (because then there is no
+/// single discriminant value).
+fn compute_discriminant_value<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    enum_type_and_layout: TyAndLayout<'tcx>,
+    variant_index: VariantIdx,
+) -> DiscrResult {
+    match enum_type_and_layout.layout.variants() {
+        &Variants::Single { .. } | &Variants::Empty => DiscrResult::NoDiscriminant,
+        &Variants::Multiple {
+            tag_encoding: TagEncoding::Direct,
+            ..
+        } => DiscrResult::Value(
+            enum_type_and_layout
+                .ty
+                .discriminant_for_variant(cx.tcx, variant_index)
+                .unwrap()
+                .val,
+        ),
+        &Variants::Multiple {
+            tag_encoding:
+                TagEncoding::Niche {
+                    ref niche_variants,
+                    niche_start,
+                    untagged_variant,
+                },
+            tag,
+            ..
+        } => {
+            if variant_index == untagged_variant {
+                let valid_range = enum_type_and_layout
+                    .for_variant(cx, variant_index)
+                    .largest_niche
+                    .as_ref()
+                    .unwrap()
+                    .valid_range;
+
+                let min = valid_range.start.min(valid_range.end);
+                let min = tag.size(cx).truncate(min);
+
+                let max = valid_range.start.max(valid_range.end);
+                let max = tag.size(cx).truncate(max);
+
+                DiscrResult::Range(min, max)
+            } else {
+                let value = (variant_index.as_u32() as u128)
+                    .wrapping_sub(niche_variants.start().as_u32() as u128)
+                    .wrapping_add(niche_start);
+                let value = tag.size(cx).truncate(value);
+                DiscrResult::Value(value)
+            }
+        }
+    }
+}
diff --git a/crates/rustc_codegen_nvvm/src/debug_info/metadata/type_map.rs b/crates/rustc_codegen_nvvm/src/debug_info/metadata/type_map.rs
new file mode 100644
index 00000000..e258ec00
--- /dev/null
+++ b/crates/rustc_codegen_nvvm/src/debug_info/metadata/type_map.rs
@@ -0,0 +1,311 @@
+use std::cell::RefCell;
+
+use rustc_abi::{Align, Size, VariantIdx};
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_macros::HashStable;
+use rustc_middle::bug;
+use rustc_middle::ty::{self, ExistentialTraitRef, Ty, TyCtxt};
+
+use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata};
+use crate::common::AsCCharPtr;
+use crate::context::CodegenCx;
+use crate::debug_info::util::{DIB, create_DIArray, debug_context};
+use crate::llvm;
+use crate::llvm::debuginfo::{DIFlags, DIScope, DIType};
+
+mod private {
+    use rustc_macros::HashStable;
+
+    // This type cannot be constructed outside of this module because
+    // it has a private field. We make use of this in order to prevent
+    // `UniqueTypeId` from being constructed directly, without asserting
+    // the preconditions.
+    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)]
+    pub struct HiddenZst;
+}
+
+/// A unique identifier for anything that we create a debuginfo node for.
+/// The types it contains are expected to already be normalized (which
+/// is debug_asserted in the constructors).
+///
+/// Note that there are some things that only show up in debuginfo, like
+/// the separate type descriptions for each enum variant. These get an ID
+/// too because they have their own debuginfo node in LLVM IR.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)]
+pub(super) enum UniqueTypeId<'tcx> {
+    /// The ID of a regular type as it shows up at the language level.
+    Ty(Ty<'tcx>, private::HiddenZst),
+    /// The ID for the single DW_TAG_variant_part nested inside the top-level
+    /// DW_TAG_structure_type that describes enums and generators.
+    VariantPart(Ty<'tcx>, private::HiddenZst),
+    /// The ID for the artificial struct type describing a single enum variant.
+    VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst),
+    /// The ID of the artificial type we create for VTables.
+    VTableTy(
+        Ty<'tcx>,
+        Option<ExistentialTraitRef<'tcx>>,
+        private::HiddenZst,
+    ),
+}
+
+impl<'tcx> UniqueTypeId<'tcx> {
+    pub fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self {
+        assert_eq!(
+            t,
+            tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), t)
+        );
+        UniqueTypeId::Ty(t, private::HiddenZst)
+    }
+
+    pub fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self {
+        assert_eq!(
+            enum_ty,
+            tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), enum_ty)
+        );
+        UniqueTypeId::VariantPart(enum_ty, private::HiddenZst)
+    }
+
+    pub fn for_enum_variant_struct_type(
+        tcx: TyCtxt<'tcx>,
+        enum_ty: Ty<'tcx>,
+        variant_idx: VariantIdx,
+    ) -> Self {
+        assert_eq!(
+            enum_ty,
+            tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), enum_ty)
+        );
+        UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst)
+    }
+
+    pub fn for_vtable_ty(
+        tcx: TyCtxt<'tcx>,
+        self_type: Ty<'tcx>,
+        implemented_trait: Option<ExistentialTraitRef<'tcx>>,
+    ) -> Self {
+        assert_eq!(
+            self_type,
+            tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), self_type)
+        );
+        assert_eq!(
+            implemented_trait,
+            tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), implemented_trait)
+        );
+        UniqueTypeId::VTableTy(self_type, implemented_trait, private::HiddenZst)
+    }
+
+    /// Generates a string version of this [UniqueTypeId], which can be used as the `UniqueId`
+    /// argument of the various `LLVMRustDIBuilderCreate*Type()` methods.
+    ///
+    /// Right now this takes the form of a hex-encoded opaque hash value.
+    pub fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String {
+        let mut hasher = StableHasher::new();
+        tcx.with_stable_hashing_context(|mut hcx| {
+            hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher))
+        });
+        hasher.finish::<Fingerprint>().to_hex()
+    }
+
+    pub fn expect_ty(self) -> Ty<'tcx> {
+        match self {
+            UniqueTypeId::Ty(ty, _) => ty,
+            _ => bug!("Expected `UniqueTypeId::Ty` but found `{:?}`", self),
+        }
+    }
+}
+
+/// The `TypeMap` is where the debug context holds the type metadata nodes
+/// created so far. The debuginfo nodes are identified by `UniqueTypeId`.
+#[derive(Default)]
+pub(crate) struct TypeMap<'ll, 'tcx> {
+    pub(super) unique_id_to_di_node: RefCell<FxHashMap<UniqueTypeId<'tcx>, &'ll DIType>>,
+}
+
+impl<'ll, 'tcx> TypeMap<'ll, 'tcx> {
+    /// Adds a `UniqueTypeId` to metadata mapping to the `TypeMap`. The method will
+    /// fail if the mapping already exists.
+    pub(super) fn insert(&self, unique_type_id: UniqueTypeId<'tcx>, metadata: &'ll DIType) {
+        if self
+            .unique_id_to_di_node
+            .borrow_mut()
+            .insert(unique_type_id, metadata)
+            .is_some()
+        {
+            bug!(
+                "type metadata for unique ID '{:?}' is already in the `TypeMap`!",
+                unique_type_id
+            );
+        }
+    }
+
+    pub(super) fn di_node_for_unique_id(
+        &self,
+        unique_type_id: UniqueTypeId<'tcx>,
+    ) -> Option<&'ll DIType> {
+        self.unique_id_to_di_node
+            .borrow()
+            .get(&unique_type_id)
+            .cloned()
+    }
+}
+
+pub(crate) struct DINodeCreationResult<'ll> {
+    pub di_node: &'ll DIType,
+    pub already_stored_in_typemap: bool,
+}
+
+impl<'ll> DINodeCreationResult<'ll> {
+    pub(crate) fn new(di_node: &'ll DIType, already_stored_in_typemap: bool) -> Self {
+        DINodeCreationResult {
+            di_node,
+            already_stored_in_typemap,
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub(crate) enum Stub<'ll> {
+    Struct,
+    Union,
+    VTableTy { vtable_holder: &'ll DIType },
+}
+
+pub(crate) struct StubInfo<'ll, 'tcx> {
+    metadata: &'ll DIType,
+    unique_type_id: UniqueTypeId<'tcx>,
+}
+
+impl<'ll, 'tcx> StubInfo<'ll, 'tcx> {
+    pub(super) fn new(
+        cx: &CodegenCx<'ll, 'tcx>,
+        unique_type_id: UniqueTypeId<'tcx>,
+        build: impl FnOnce(&CodegenCx<'ll, 'tcx>, /* unique_type_id_str: */ &str) -> &'ll DIType,
+    ) -> StubInfo<'ll, 'tcx> {
+        let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx);
+        let di_node = build(cx, &unique_type_id_str);
+        StubInfo {
+            metadata: di_node,
+            unique_type_id,
+        }
+    }
+}
+
+/// Create a stub debuginfo node onto which fields and nested types can be attached.
+pub(super) fn stub<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    kind: Stub<'ll>,
+    unique_type_id: UniqueTypeId<'tcx>,
+    name: &str,
+    def_location: Option<DefinitionLocation<'ll>>,
+    (size, align): (Size, Align),
+    containing_scope: Option<&'ll DIScope>,
+    flags: DIFlags,
+) -> StubInfo<'ll, 'tcx> {
+    let empty_array = create_DIArray(DIB(cx), &[]);
+    let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx);
+
+    let (file_metadata, line_number) = if let Some(def_location) = def_location {
+        (def_location.0, def_location.1)
+    } else {
+        (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)
+    };
+
+    let metadata = match kind {
+        Stub::Struct | Stub::VTableTy { .. } => {
+            let vtable_holder = match kind {
+                Stub::VTableTy { vtable_holder } => Some(vtable_holder),
+                _ => None,
+            };
+            unsafe {
+                llvm::LLVMRustDIBuilderCreateStructType(
+                    DIB(cx),
+                    containing_scope,
+                    name.as_c_char_ptr(),
+                    name.len(),
+                    file_metadata,
+                    line_number,
+                    size.bits(),
+                    align.bits() as u32,
+                    flags,
+                    None,
+                    empty_array,
+                    0,
+                    vtable_holder,
+                    unique_type_id_str.as_c_char_ptr(),
+                    unique_type_id_str.len(),
+                )
+            }
+        }
+        Stub::Union => unsafe {
+            llvm::LLVMRustDIBuilderCreateUnionType(
+                DIB(cx),
+                containing_scope,
+                name.as_c_char_ptr(),
+                name.len(),
+                unknown_file_metadata(cx),
+                UNKNOWN_LINE_NUMBER,
+                size.bits(),
+                align.bits() as u32,
+                flags,
+                Some(empty_array),
+                0,
+                unique_type_id_str.as_c_char_ptr(),
+                unique_type_id_str.len(),
+            )
+        },
+    };
+    StubInfo {
+        metadata,
+        unique_type_id,
+    }
+}
+
+/// This function enables creating debuginfo nodes that can recursively refer to themselves.
+/// It will first insert the given stub into the type map and only then execute the `members`
+/// and `generics` closures passed in. These closures have access to the stub so they can
+/// directly attach fields to them. If the type of a field transitively refers back
+/// to the type currently being built, the stub will already be found in the type map,
+/// which effectively breaks the recursion cycle.
+pub(super) fn build_type_with_children<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    stub_info: StubInfo<'ll, 'tcx>,
+    members: impl FnOnce(&CodegenCx<'ll, 'tcx>, &'ll DIType) -> SmallVec<&'ll DIType>,
+    generics: impl FnOnce(&CodegenCx<'ll, 'tcx>) -> SmallVec<&'ll DIType>,
+) -> DINodeCreationResult<'ll> {
+    assert_eq!(
+        debug_context(cx)
+            .type_map
+            .di_node_for_unique_id(stub_info.unique_type_id),
+        None
+    );
+
+    debug_context(cx)
+        .type_map
+        .insert(stub_info.unique_type_id, stub_info.metadata);
+
+    let members: SmallVec<_> = members(cx, stub_info.metadata)
+        .into_iter()
+        .map(|node| Some(node))
+        .collect();
+    let generics: SmallVec<Option<&'ll DIType>> =
+        generics(cx).into_iter().map(|node| Some(node)).collect();
+
+    if !(members.is_empty() && generics.is_empty()) {
+        unsafe {
+            let members_array = create_DIArray(DIB(cx), &members[..]);
+            let generics_array = create_DIArray(DIB(cx), &generics[..]);
+            llvm::LLVMRustDICompositeTypeReplaceArrays(
+                DIB(cx),
+                stub_info.metadata,
+                Some(members_array),
+                Some(generics_array),
+            );
+        }
+    }
+
+    DINodeCreationResult {
+        di_node: stub_info.metadata,
+        already_stored_in_typemap: true,
+    }
+}
diff --git a/crates/rustc_codegen_nvvm/src/debug_info/mod.rs b/crates/rustc_codegen_nvvm/src/debug_info/mod.rs
index 23e7f9a9..c370b627 100644
--- a/crates/rustc_codegen_nvvm/src/debug_info/mod.rs
+++ b/crates/rustc_codegen_nvvm/src/debug_info/mod.rs
@@ -1,36 +1,43 @@
+use std::cell::OnceCell;
 use std::cell::RefCell;
 use std::ffi::CString;
+use std::iter;
+use std::ops::Range;
+use std::sync::Arc;
 
 use libc::c_uint;
+use rustc_abi::Size;
 use rustc_codegen_ssa::debuginfo::type_names;
 use rustc_codegen_ssa::mir::debuginfo::VariableKind::*;
 use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind};
 use rustc_codegen_ssa::traits::*;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::unord::UnordMap;
 use rustc_hir::def_id::{DefId, DefIdMap};
-use rustc_index::vec::IndexVec;
+use rustc_index::IndexVec;
 use rustc_middle::mir;
-use rustc_middle::ty::layout::HasTyCtxt;
-use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
-use rustc_middle::ty::{self, Binder, ExistentialTraitRef, Instance, ParamEnv, Ty, TypeFoldable};
+use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv};
+use rustc_middle::ty::{self, GenericArgKind, GenericArgsRef, Instance, Ty, TypeVisitableExt};
 use rustc_session::config::{self, DebugInfo};
 use rustc_span::symbol::Symbol;
-use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, Span};
-use rustc_target::abi::call::FnAbi;
-use rustc_target::abi::{Primitive, Size};
+use rustc_span::{
+    self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId,
+};
+use rustc_target::callconv::FnAbi;
+use smallvec::SmallVec;
 
 use crate::builder::Builder;
+use crate::common::AsCCharPtr;
 use crate::context::CodegenCx;
 use crate::debug_info::util::{create_DIArray, is_node_local_to_unit};
-use crate::llvm::{self, debuginfo::*, Value};
+use crate::llvm::{self, Value, debuginfo::*};
 
 use self::namespace::*;
 use self::util::DIB;
 use create_scope_map::compute_mir_scopes;
 
 mod create_scope_map;
-mod metadata;
+mod dwarf_const;
+pub(crate) mod metadata;
 mod namespace;
 mod util;
 
@@ -41,24 +48,19 @@ const DW_TAG_auto_variable: c_uint = 0x100;
 #[allow(non_upper_case_globals)]
 const DW_TAG_arg_variable: c_uint = 0x101;
 
-pub struct CrateDebugContext<'a, 'tcx> {
+pub struct CodegenUnitDebugContext<'ll, 'tcx> {
     #[allow(dead_code)]
-    llcontext: &'a llvm::Context,
-    llmod: &'a llvm::Module,
-    builder: &'a mut DIBuilder<'a>,
-    #[allow(clippy::type_complexity)]
-    created_files: RefCell<FxHashMap<(Option<String>, Option<String>), &'a DIFile>>,
-    created_enum_disr_types: RefCell<FxHashMap<(DefId, Primitive), &'a DIType>>,
-
-    type_map: RefCell<TypeMap<'a, 'tcx>>,
-    namespace_map: RefCell<DefIdMap<&'a DIScope>>,
-
-    // This collection is used to assert that composite types (structs, enums,
-    // ...) have their members only set once:
-    composite_types_completed: RefCell<FxHashSet<&'a DIType>>,
+    llcontext: &'ll llvm::Context,
+    llmod: &'ll llvm::Module,
+    builder: &'ll mut DIBuilder<'ll>,
+    created_files: RefCell<UnordMap<Option<(StableSourceFileId, SourceFileHash)>, &'ll DIFile>>,
+
+    type_map: metadata::TypeMap<'ll, 'tcx>,
+    namespace_map: RefCell<DefIdMap<&'ll DIScope>>,
+    recursion_marker_type: OnceCell<&'ll DIType>,
 }
 
-impl<'a, 'tcx> Drop for CrateDebugContext<'a, 'tcx> {
+impl<'a, 'tcx> Drop for CodegenUnitDebugContext<'a, 'tcx> {
     fn drop(&mut self) {
         unsafe {
             llvm::LLVMRustDIBuilderDispose(&mut *(self.builder as *mut _));
@@ -66,38 +68,43 @@ impl<'a, 'tcx> Drop for CrateDebugContext<'a, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> CrateDebugContext<'a, 'tcx> {
+impl<'a, 'tcx> CodegenUnitDebugContext<'a, 'tcx> {
     pub(crate) fn new(llmod: &'a llvm::Module) -> Self {
         let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) };
         // DIBuilder inherits context from the module, so we'd better use the same one
         let llcontext = unsafe { llvm::LLVMGetModuleContext(llmod) };
-        CrateDebugContext {
+        CodegenUnitDebugContext {
             llcontext,
             llmod,
             builder,
             created_files: Default::default(),
-            created_enum_disr_types: Default::default(),
             type_map: Default::default(),
             namespace_map: RefCell::new(Default::default()),
-            composite_types_completed: Default::default(),
+            recursion_marker_type: OnceCell::new(),
         }
     }
 
-    pub fn finalize(&self) {
+    pub(crate) fn finalize(&self) {
         unsafe {
             llvm::LLVMRustDIBuilderFinalize(self.builder);
 
             // Prevent bitcode readers from deleting the debug info.
-            let ptr = "Debug Info Version\0".as_ptr();
             llvm::LLVMRustAddModuleFlag(
                 self.llmod,
-                ptr.cast(),
+                c"Debug Info Version".as_ptr(),
                 llvm::LLVMRustDebugMetadataVersion(),
             );
         }
     }
 }
 
+/// Creates any deferred debug metadata nodes
+pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) {
+    if let Some(dbg_cx) = &cx.dbg_cx {
+        dbg_cx.finalize();
+    }
+}
+
 impl<'a, 'll, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'll, 'tcx> {
     fn dbg_var_addr(
         &mut self,
@@ -106,23 +113,32 @@ impl<'a, 'll, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'll, 'tcx> {
         variable_alloca: &'ll Value,
         direct_offset: Size,
         indirect_offsets: &[Size],
+        fragment: Option<Range<Size>>,
     ) {
-        let op_deref = || unsafe { llvm::LLVMRustDIBuilderCreateOpDeref() };
-        let op_plus_uconst = || unsafe { llvm::LLVMRustDIBuilderCreateOpPlusUconst() };
-        let mut addr_ops = Vec::new();
+        use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst};
+
+        let mut addr_ops = SmallVec::<[i64; 8]>::new();
 
         if direct_offset.bytes() > 0 {
-            addr_ops.push(op_plus_uconst());
+            addr_ops.push(DW_OP_plus_uconst);
             addr_ops.push(direct_offset.bytes() as i64);
         }
         for &offset in indirect_offsets {
-            addr_ops.push(op_deref());
+            addr_ops.push(DW_OP_deref);
             if offset.bytes() > 0 {
-                addr_ops.push(op_plus_uconst());
+                addr_ops.push(DW_OP_plus_uconst);
                 addr_ops.push(offset.bytes() as i64);
             }
         }
 
+        if let Some(fragment) = fragment {
+            // `DW_OP_LLVM_fragment` takes as arguments the fragment's
+            // offset and size, both of them in bits.
+            addr_ops.push(DW_OP_LLVM_fragment);
+            addr_ops.push(fragment.start.bits() as i64);
+            addr_ops.push((fragment.end - fragment.start).bits() as i64);
+        }
+
         unsafe {
             llvm::LLVMRustDIBuilderInsertDeclareAtEnd(
                 DIB(self.cx()),
@@ -139,7 +155,7 @@ impl<'a, 'll, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'll, 'tcx> {
     fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) {
         unsafe {
             let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc);
-            llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval);
+            llvm::LLVMSetCurrentDebugLocation(self.llbuilder, Some(dbg_loc_as_llval));
         }
     }
 
@@ -168,12 +184,22 @@ impl<'a, 'll, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'll, 'tcx> {
             llvm::set_value_name(value, name.as_bytes());
         }
     }
+
+    fn clear_dbg_loc(&mut self) {
+        unsafe {
+            llvm::LLVMSetCurrentDebugLocation(self.llbuilder, None);
+        }
+    }
+
+    fn get_dbg_loc(&self) -> Option<Self::DILocation> {
+        None // TODO: implement this
+    }
 }
 
 /// A source code location used to generate debug information.
 pub struct DebugLoc {
     /// Information about the original source file.
-    pub file: Lrc<SourceFile>,
+    pub file: Arc<SourceFile>,
     /// The (1-based) line number.
     pub line: u32,
     /// The (1-based) column number.
@@ -200,37 +226,32 @@ impl<'ll> CodegenCx<'ll, '_> {
     }
 }
 
-impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn create_function_debug_context(
         &self,
         instance: Instance<'tcx>,
         fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
-        llfn: &'ll Value,
+        llfn: Self::Function,
         mir: &mir::Body<'tcx>,
-    ) -> Option<FunctionDebugContext<&'ll DIScope, &'ll DILocation>> {
+    ) -> Option<FunctionDebugContext<'tcx, Self::DIScope, Self::DILocation>> {
         if self.sess().opts.debuginfo == DebugInfo::None {
             return None;
         }
 
         // Initialize fn debug context (including scopes).
         let empty_scope = DebugScope {
-            dbg_scope: None,
+            dbg_scope: self.dbg_scope_fn(instance, fn_abi, Some(llfn)),
             inlined_at: None,
             file_start_pos: BytePos(0),
             file_end_pos: BytePos(0),
         };
         let mut fn_debug_context = FunctionDebugContext {
             scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes),
+            inlined_function_scopes: Default::default(),
         };
 
         // Fill in all the scopes, with the information from the MIR body.
-        compute_mir_scopes(
-            self,
-            instance,
-            mir,
-            self.dbg_scope_fn(instance, fn_abi, Some(llfn)),
-            &mut fn_debug_context,
-        );
+        compute_mir_scopes(self, instance, mir, &mut fn_debug_context);
 
         Some(fn_debug_context)
     }
@@ -241,9 +262,11 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         maybe_definition_llfn: Option<&'ll Value>,
     ) -> &'ll DIScope {
+        let tcx = self.tcx();
+
         let def_id = instance.def_id();
         let containing_scope = get_containing_scope(self, instance);
-        let span = self.tcx.def_span(def_id);
+        let span = tcx.def_span(def_id);
         let loc = self.lookup_debug_loc(span.lo());
         let file_metadata = file_metadata(self, &loc.file);
 
@@ -253,16 +276,16 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         };
 
         let mut name = String::new();
-        type_names::push_item_name(self.tcx(), def_id, false, &mut name);
+        type_names::push_item_name(tcx, def_id, false, &mut name);
 
         // Find the enclosing function, in case this is a closure.
-        let enclosing_fn_def_id = self.tcx().typeck_root_def_id(def_id);
+        let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id);
 
         // Get_template_parameters() will append a `<...>` clause to the function
         // name if necessary.
-        let generics = self.tcx().generics_of(enclosing_fn_def_id);
-        let substs = instance.substs.truncate_to(self.tcx(), generics);
-        let template_parameters = get_template_parameters(self, generics, substs, &mut name);
+        let generics = tcx.generics_of(enclosing_fn_def_id);
+        let args = instance.args.truncate_to(tcx, generics);
+        let template_parameters = get_template_parameters(self, generics, args);
 
         let linkage_name = &mangled_name_of_instance(self, instance).name;
         // Omit the linkage_name if it is the same as subprogram name.
@@ -278,14 +301,14 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
 
         let mut flags = DIFlags::FlagPrototyped;
 
-        if fn_abi.ret.layout.abi.is_uninhabited() {
+        if fn_abi.ret.layout.is_uninhabited() {
             flags |= DIFlags::FlagNoReturn;
         }
 
         unsafe {
             return llvm::LLVMRustDIBuilderCreateFunction(
                 DIB(self),
-                containing_scope,
+                containing_scope.0,
                 name.as_ptr(),
                 linkage_name.as_ptr(),
                 file_metadata,
@@ -316,18 +339,14 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             signature.push(if fn_abi.ret.is_ignore() {
                 None
             } else {
-                Some(type_metadata(
-                    cx,
-                    fn_abi.ret.layout.ty,
-                    rustc_span::DUMMY_SP,
-                ))
+                Some(type_di_node(cx, fn_abi.ret.layout.ty))
             });
 
             signature.extend(
                 fn_abi
                     .args
                     .iter()
-                    .map(|arg| Some(type_metadata(cx, arg.layout.ty, rustc_span::DUMMY_SP))),
+                    .map(|arg| Some(type_di_node(cx, arg.layout.ty))),
             );
 
             create_DIArray(DIB(cx), &signature[..])
@@ -336,38 +355,27 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         fn get_template_parameters<'ll, 'tcx>(
             cx: &CodegenCx<'ll, 'tcx>,
             generics: &ty::Generics,
-            substs: SubstsRef<'tcx>,
-            name_to_append_suffix_to: &mut String,
+            args: GenericArgsRef<'tcx>,
         ) -> &'ll DIArray {
-            type_names::push_generic_params(
-                cx.tcx,
-                cx.tcx
-                    .normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs),
-                name_to_append_suffix_to,
-            );
-
-            if substs.types().next().is_none() {
+            if args.types().next().is_none() {
                 return create_DIArray(DIB(cx), &[]);
             }
 
             // Again, only create type information if full debuginfo is enabled
             let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full {
                 let names = get_parameter_names(cx, generics);
-                substs
-                    .into_iter()
-                    .zip(names.into_iter())
+                iter::zip(args, names)
                     .filter_map(|(kind, name)| {
                         if let GenericArgKind::Type(ty) = kind.unpack() {
-                            let actual_type =
-                                cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty);
-                            let actual_type_metadata =
-                                type_metadata(cx, actual_type, rustc_span::DUMMY_SP);
-                            let name = CString::new(&*name.as_str()).unwrap();
+                            let actual_type = cx.tcx.normalize_erasing_regions(cx.typing_env(), ty);
+                            let actual_type_metadata = type_di_node(cx, actual_type);
+                            let name = name.as_str();
                             Some(unsafe {
                                 Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter(
                                     DIB(cx),
                                     None,
-                                    name.as_ptr().cast(),
+                                    name.as_c_char_ptr(),
+                                    name.len(),
                                     actual_type_metadata,
                                 ))
                             })
@@ -380,71 +388,64 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 vec![]
             };
 
-            create_DIArray(DIB(cx), &template_params[..])
+            create_DIArray(DIB(cx), &template_params)
         }
 
         fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec<Symbol> {
             let mut names = generics.parent.map_or_else(Vec::new, |def_id| {
                 get_parameter_names(cx, cx.tcx.generics_of(def_id))
             });
-            names.extend(generics.params.iter().map(|param| param.name));
+            names.extend(generics.own_params.iter().map(|param| param.name));
             names
         }
 
         fn get_containing_scope<'ll, 'tcx>(
             cx: &CodegenCx<'ll, 'tcx>,
             instance: Instance<'tcx>,
-        ) -> &'ll DIScope {
+        ) -> (&'ll DIScope, bool) {
             // First, let's see if this is a method within an inherent impl. Because
             // if yes, we want to make the result subroutine DIE a child of the
             // subroutine's self-type.
-            let self_type = cx
-                .tcx
-                .impl_of_method(instance.def_id())
-                .and_then(|impl_def_id| {
-                    // If the method does *not* belong to a trait, proceed
-                    if cx.tcx.trait_id_of_impl(impl_def_id).is_none() {
-                        let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions(
-                            instance.substs,
-                            ty::ParamEnv::reveal_all(),
-                            cx.tcx.type_of(impl_def_id),
-                        );
-
-                        // Only "class" methods are generally understood by LLVM,
-                        // so avoid methods on other types (e.g., `<*mut T>::null`).
-                        match impl_self_ty.kind() {
-                            ty::Adt(def, ..) if !def.is_box() => {
-                                // Again, only create type information if full debuginfo is enabled
-                                if cx.sess().opts.debuginfo == DebugInfo::Full
-                                    && !impl_self_ty.definitely_needs_subst(cx.tcx)
-                                {
-                                    Some(type_metadata(cx, impl_self_ty, rustc_span::DUMMY_SP))
-                                } else {
-                                    Some(namespace::item_namespace(cx, def.did))
-                                }
-                            }
-                            _ => None,
+            if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) {
+                // If the method does *not* belong to a trait, proceed
+                if cx.tcx.trait_id_of_impl(impl_def_id).is_none() {
+                    let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions(
+                        instance.args,
+                        cx.typing_env(),
+                        cx.tcx.type_of(impl_def_id),
+                    );
+
+                    // Only "class" methods are generally understood by LLVM,
+                    // so avoid methods on other types (e.g., `<*mut T>::null`).
+                    if let ty::Adt(def, ..) = impl_self_ty.kind()
+                        && !def.is_box()
+                    {
+                        // Again, only create type information if full debuginfo is enabled
+                        if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param()
+                        {
+                            return (type_di_node(cx, impl_self_ty), true);
+                        } else {
+                            return (namespace::item_namespace(cx, def.did()), false);
                         }
-                    } else {
-                        // For trait method impls we still use the "parallel namespace"
-                        // strategy
-                        None
                     }
-                });
-
-            self_type.unwrap_or_else(|| {
-                namespace::item_namespace(
-                    cx,
-                    DefId {
-                        krate: instance.def_id().krate,
-                        index: cx
-                            .tcx
-                            .def_key(instance.def_id())
-                            .parent
-                            .expect("get_containing_scope: missing parent?"),
-                    },
-                )
-            })
+                } else {
+                    // For trait method impls we still use the "parallel namespace"
+                    // strategy
+                }
+            }
+            let scope = namespace::item_namespace(
+                cx,
+                DefId {
+                    krate: instance.def_id().krate,
+                    index: cx
+                        .tcx
+                        .def_key(instance.def_id())
+                        .parent
+                        .expect("get_containing_scope: missing parent?"),
+                },
+            );
+
+            (scope, false)
         }
     }
 
@@ -459,13 +460,13 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         unsafe { llvm::LLVMRustDIBuilderCreateDebugLocation(line, col, scope, inlined_at) }
     }
 
-    fn create_vtable_metadata(
+    fn create_vtable_debuginfo(
         &self,
         ty: Ty<'tcx>,
-        _: Option<Binder<'tcx, ExistentialTraitRef<'tcx>>>,
+        trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
         vtable: Self::Value,
     ) {
-        metadata::create_vtable_metadata(self, ty, vtable)
+        metadata::create_vtable_di_node(self, ty, trait_ref, vtable)
     }
 
     fn extend_scope_to_file(
@@ -477,9 +478,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     }
 
     fn debuginfo_finalize(&self) {
-        if let Some(dbg_cx) = &self.dbg_cx {
-            dbg_cx.finalize();
-        }
+        finalize(self)
     }
 
     fn create_dbg_var(
@@ -493,7 +492,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         let loc = self.lookup_debug_loc(span.lo());
         let file_metadata = file_metadata(self, &loc.file);
 
-        let type_metadata = type_metadata(self, variable_type, span);
+        let type_metadata = type_di_node(self, variable_type);
 
         let (argument_index, dwarf_tag) = match variable_kind {
             ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable),
diff --git a/crates/rustc_codegen_nvvm/src/debug_info/namespace.rs b/crates/rustc_codegen_nvvm/src/debug_info/namespace.rs
index 75f198c1..a0bb84dc 100644
--- a/crates/rustc_codegen_nvvm/src/debug_info/namespace.rs
+++ b/crates/rustc_codegen_nvvm/src/debug_info/namespace.rs
@@ -1,16 +1,15 @@
 // Namespace Handling.
 
-use std::ffi::CString;
-
 use rustc_codegen_ssa::debuginfo::type_names;
 use rustc_middle::ty::{self, Instance};
 
+use crate::common::AsCCharPtr;
 use crate::context::CodegenCx;
 use crate::llvm;
 use crate::llvm::debuginfo::DIScope;
 use rustc_hir::def_id::DefId;
 
-use super::util::{debug_context, DIB};
+use super::util::{DIB, debug_context};
 
 pub(crate) fn mangled_name_of_instance<'a, 'tcx>(
     cx: &CodegenCx<'a, 'tcx>,
@@ -36,18 +35,18 @@ pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'l
         )
     });
 
-    let namespace_name_string = CString::new({
+    let namespace_name_string = {
         let mut output = String::new();
         type_names::push_item_name(cx.tcx, def_id, false, &mut output);
         output
-    })
-    .unwrap();
+    };
 
     let scope = unsafe {
         llvm::LLVMRustDIBuilderCreateNameSpace(
             DIB(cx),
             parent_scope,
-            namespace_name_string.as_ptr(),
+            namespace_name_string.as_c_char_ptr(),
+            namespace_name_string.len(),
         )
     };
 
diff --git a/crates/rustc_codegen_nvvm/src/debug_info/util.rs b/crates/rustc_codegen_nvvm/src/debug_info/util.rs
index 10a742f3..742d0dd8 100644
--- a/crates/rustc_codegen_nvvm/src/debug_info/util.rs
+++ b/crates/rustc_codegen_nvvm/src/debug_info/util.rs
@@ -1,12 +1,14 @@
 use rustc_hir::def_id::DefId;
-use rustc_middle::ty::DefIdTree;
+use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
+use rustc_middle::ty::{self, Ty};
+use tracing::trace;
 
 use crate::context::CodegenCx;
 use crate::llvm;
 use crate::llvm::debuginfo::{DIArray, DIBuilder, DIDescriptor, DIScope};
 
+use super::CodegenUnitDebugContext;
 use super::namespace::item_namespace;
-use super::CrateDebugContext;
 
 pub(crate) fn is_node_local_to_unit(cx: &CodegenCx<'_, '_>, def_id: DefId) -> bool {
     // The is_local_to_unit flag indicates whether a function is local to the
@@ -31,7 +33,7 @@ pub(crate) fn create_DIArray<'ll>(
 #[inline]
 pub(crate) fn debug_context<'a, 'll, 'tcx>(
     cx: &'a CodegenCx<'ll, 'tcx>,
-) -> &'a CrateDebugContext<'ll, 'tcx> {
+) -> &'a CodegenUnitDebugContext<'ll, 'tcx> {
     cx.dbg_cx.as_ref().unwrap()
 }
 
@@ -45,10 +47,53 @@ pub(crate) fn get_namespace_for_item<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     def_id: DefId,
 ) -> &'ll DIScope {
-    item_namespace(
-        cx,
-        cx.tcx
-            .parent(def_id)
-            .expect("get_namespace_for_item: missing parent?"),
-    )
+    item_namespace(cx, cx.tcx.parent(def_id))
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub(crate) enum WidePtrKind {
+    Slice,
+    Dyn,
+}
+
+/// Determines if `pointee_ty` is slice-like or trait-object-like, i.e.
+/// if the second field of the wide pointer is a length or a vtable-pointer.
+/// If `pointee_ty` does not require a wide pointer (because it is Sized) then
+/// the function returns `None`.
+pub(crate) fn wide_pointer_kind<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    pointee_ty: Ty<'tcx>,
+) -> Option<WidePtrKind> {
+    let pointee_tail_ty = cx.tcx.struct_tail_for_codegen(pointee_ty, cx.typing_env());
+    let layout = cx.layout_of(pointee_tail_ty);
+    trace!(
+        "wide_pointer_kind: {:?} has layout {:?} (is_unsized? {})",
+        pointee_tail_ty,
+        layout,
+        layout.is_unsized()
+    );
+
+    if layout.is_sized() {
+        return None;
+    }
+
+    match *pointee_tail_ty.kind() {
+        ty::Str | ty::Slice(_) => Some(WidePtrKind::Slice),
+        ty::Dynamic(..) => Some(WidePtrKind::Dyn),
+        ty::Foreign(_) => {
+            // Assert that pointers to foreign types really are thin:
+            assert_eq!(
+                cx.size_of(Ty::new_imm_ptr(cx.tcx, pointee_tail_ty)),
+                cx.size_of(Ty::new_imm_ptr(cx.tcx, cx.tcx.types.u8))
+            );
+            None
+        }
+        _ => {
+            // For all other pointee types we should already have returned None
+            // at the beginning of the function.
+            panic!(
+                "wide_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {pointee_tail_ty:?}"
+            )
+        }
+    }
 }
diff --git a/crates/rustc_codegen_nvvm/src/init.rs b/crates/rustc_codegen_nvvm/src/init.rs
index 2a4a346d..3b13b8a3 100644
--- a/crates/rustc_codegen_nvvm/src/init.rs
+++ b/crates/rustc_codegen_nvvm/src/init.rs
@@ -1,15 +1,14 @@
-use libc::c_int;
-use rustc_metadata::dynamic_lib::DynamicLibrary;
-use rustc_middle::bug;
-use rustc_session::Session;
-use rustc_target::spec::MergeFunctions;
 use std::ffi::CString;
-
 use std::mem;
 use std::path::Path;
 use std::str;
-use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Once;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use libc::c_int;
+use rustc_middle::bug;
+use rustc_session::Session;
+use rustc_target::spec::MergeFunctions;
 
 use crate::llvm;
 
@@ -75,19 +74,20 @@ unsafe fn configure_llvm(sess: &Session) {
         };
         // Set the llvm "program name" to make usage and invalid argument messages more clear.
         // add("rustc -Cllvm-args=\"...\" with", true);
-        if sess.time_llvm_passes() {
+
+        if sess.opts.unstable_opts.time_llvm_passes {
             add("-time-passes", false);
         }
-        if sess.print_llvm_passes() {
+        if sess.opts.unstable_opts.print_llvm_passes {
             add("-debug-pass=Structure", false);
         }
-        if !sess.opts.debugging_opts.no_generate_arange_section {
+        if !sess.opts.unstable_opts.no_generate_arange_section {
             add("-generate-arange-section", false);
         }
 
         match sess
             .opts
-            .debugging_opts
+            .unstable_opts
             .merge_functions
             .unwrap_or(sess.target.merge_functions)
         {
@@ -112,11 +112,11 @@ unsafe fn configure_llvm(sess: &Session) {
         // }
     }
 
-    llvm::LLVMInitializePasses();
+    unsafe { llvm::LLVMInitializePasses() };
 
-    for plugin in &sess.opts.debugging_opts.llvm_plugins {
+    for plugin in &sess.opts.unstable_opts.llvm_plugins {
         let path = Path::new(plugin);
-        let res = DynamicLibrary::open(path);
+        let res = unsafe { libloading::Library::new(path) };
         match res {
             Ok(_) => {}
             Err(e) => bug!("couldn't load plugin: {}", e),
@@ -124,10 +124,12 @@ unsafe fn configure_llvm(sess: &Session) {
         mem::forget(res);
     }
 
-    llvm::LLVMInitializeNVPTXTarget();
-    llvm::LLVMInitializeNVPTXTargetInfo();
-    llvm::LLVMInitializeNVPTXTargetMC();
-    llvm::LLVMInitializeNVPTXAsmPrinter();
+    unsafe {
+        llvm::LLVMInitializeNVPTXTarget();
+        llvm::LLVMInitializeNVPTXTargetInfo();
+        llvm::LLVMInitializeNVPTXTargetMC();
+        llvm::LLVMInitializeNVPTXAsmPrinter();
 
-    llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr());
+        llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr());
+    }
 }
diff --git a/crates/rustc_codegen_nvvm/src/int_replace.rs b/crates/rustc_codegen_nvvm/src/int_replace.rs
index 03f93eeb..122eb143 100644
--- a/crates/rustc_codegen_nvvm/src/int_replace.rs
+++ b/crates/rustc_codegen_nvvm/src/int_replace.rs
@@ -1,4 +1,3 @@
-use rustc_codegen_ssa::traits::BaseTypeMethods;
 use tracing::trace;
 
 use crate::builder::unnamed;
@@ -52,9 +51,7 @@ pub(crate) fn get_transformed_type<'ll, 'tcx>(
                     let int_ty = LLVMIntTypeInContext(cx.llcx, width);
                     trace!(
                         "Transforming irregular int type `{:?}` to vector ty `{:?}` with length {}",
-                        ty,
-                        int_ty,
-                        count
+                        ty, int_ty, count
                     );
                     (LLVMVectorType(int_ty, count), true)
                 }
diff --git a/crates/rustc_codegen_nvvm/src/intrinsic.rs b/crates/rustc_codegen_nvvm/src/intrinsic.rs
index 4359f4be..7fceb702 100644
--- a/crates/rustc_codegen_nvvm/src/intrinsic.rs
+++ b/crates/rustc_codegen_nvvm/src/intrinsic.rs
@@ -1,23 +1,27 @@
-use crate::abi::LlvmType;
-use crate::llvm::{self, Value};
-use crate::target;
-use crate::ty::LayoutLlvmExt;
-use crate::{builder::Builder, context::CodegenCx};
-use rustc_codegen_ssa::common::span_invalid_monomorphization_error;
-use rustc_codegen_ssa::mir::place::PlaceRef;
-use rustc_codegen_ssa::traits::DerivedTypeMethods;
-use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods, ConstMethods, OverflowOp};
-use rustc_codegen_ssa::{mir::operand::OperandRef, traits::IntrinsicCallMethods};
-use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
-use rustc_middle::ty::Ty;
-use rustc_middle::{bug, ty};
+use rustc_abi as abi;
+use rustc_abi::{self, Float, HasDataLayout, Primitive};
+use rustc_codegen_ssa::errors::InvalidMonomorphization;
+use rustc_codegen_ssa::mir::operand::OperandValue;
+use rustc_codegen_ssa::mir::place::PlaceValue;
+use rustc_codegen_ssa::mir::{operand::OperandRef, place::PlaceRef};
+use rustc_codegen_ssa::traits::{
+    BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, IntrinsicCallBuilderMethods,
+    OverflowOp,
+};
+use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
+use rustc_middle::ty::{self, Ty};
+use rustc_middle::{bug, span_bug};
 use rustc_span::symbol::kw;
-use rustc_span::Span;
-use rustc_span::{sym, Symbol};
-use rustc_target::abi::call::{FnAbi, PassMode};
-use rustc_target::abi::{self, HasDataLayout, Primitive};
+use rustc_span::{Span, Symbol, sym};
+use rustc_target::callconv::{FnAbi, PassMode};
 use tracing::trace;
 
+use crate::abi::LlvmType;
+use crate::builder::Builder;
+use crate::context::CodegenCx;
+use crate::llvm::{self, Metadata, Type, Value};
+use crate::ty::LayoutLlvmExt;
+
 // libnvvm does not support some advanced intrinsics for i128 so we just abort on them for now. In the future
 // we should emulate them in software.
 fn handle_128_bit_intrinsic<'a, 'll, 'tcx>(b: &mut Builder<'a, 'll, 'tcx>) -> &'ll Value {
@@ -35,21 +39,21 @@ fn saturating_intrinsic_impl<'a, 'll, 'tcx>(
 ) -> &'ll Value {
     use rustc_middle::ty::IntTy::*;
     use rustc_middle::ty::UintTy::*;
-    use rustc_middle::ty::{Int, Uint};
-
-    let ty = b.cx.tcx().mk_ty(match (signed, width) {
-        (true, 8) => Int(I8),
-        (true, 16) => Int(I16),
-        (true, 32) => Int(I32),
-        (true, 64) => Int(I64),
-        (true, 128) => Int(I128),
-        (false, 8) => Uint(U8),
-        (false, 16) => Uint(U16),
-        (false, 32) => Uint(U32),
-        (false, 64) => Uint(U64),
-        (false, 128) => Uint(U128),
+
+    let tcx = b.tcx;
+    let ty = match (signed, width) {
+        (true, 8) => Ty::new_int(tcx, I8),
+        (true, 16) => Ty::new_int(tcx, I16),
+        (true, 32) => Ty::new_int(tcx, I32),
+        (true, 64) => Ty::new_int(tcx, I64),
+        (true, 128) => Ty::new_int(tcx, I128),
+        (false, 8) => Ty::new_uint(tcx, U8),
+        (false, 16) => Ty::new_uint(tcx, U16),
+        (false, 32) => Ty::new_uint(tcx, U32),
+        (false, 64) => Ty::new_uint(tcx, U64),
+        (false, 128) => Ty::new_uint(tcx, U128),
         _ => unreachable!(),
-    });
+    };
 
     let unsigned_max_value = match width {
         8 => u8::MAX as i64,
@@ -99,7 +103,10 @@ fn saturating_intrinsic_impl<'a, 'll, 'tcx>(
     }
 }
 
-fn get_simple_intrinsic<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, name: Symbol) -> Option<&'ll Value> {
+fn get_simple_intrinsic<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    name: Symbol,
+) -> Option<(&'ll Type, &'ll Value)> {
     #[rustfmt::skip]
     let llvm_name = match name {
         sym::sqrtf32      => "__nv_sqrtf",
@@ -138,27 +145,17 @@ fn get_simple_intrinsic<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, name: Symbol) -> O
         sym::ceilf64      => "__nv_ceil",
         sym::truncf32     => "__nv_truncf",
         sym::truncf64     => "__nv_trunc",
-        sym::rintf32      => "__nv_rintf",
-        sym::rintf64      => "__nv_rint",
-        sym::nearbyintf32 => "__nv_nearbyintf",
-        sym::nearbyintf64 => "__nv_nearbyint",
         sym::roundf32     => "__nv_roundf",
         sym::roundf64     => "__nv_round",
+        sym::round_ties_even_f32 => "__nv_rintf",
+        sym::round_ties_even_f64 => "__nv_rint",
         _ => return None,
     };
     trace!("Retrieving nv intrinsic `{:?}`", llvm_name);
     Some(cx.get_intrinsic(llvm_name))
 }
 
-fn int_type_width_signed(ty: Ty<'_>, _cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> {
-    match ty.kind() {
-        ty::Int(t) => Some((t.bit_width().unwrap_or(target::POINTER_WIDTH as u64), true)),
-        ty::Uint(t) => Some((t.bit_width().unwrap_or(target::POINTER_WIDTH as u64), false)),
-        _ => None,
-    }
-}
-
-impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
+impl<'a, 'll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
     fn codegen_intrinsic_call(
         &mut self,
         instance: ty::Instance<'tcx>,
@@ -166,17 +163,16 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         args: &[OperandRef<'tcx, &'ll Value>],
         llresult: &'ll Value,
         span: Span,
-    ) {
+    ) -> Result<(), ty::Instance<'tcx>> {
         let tcx = self.tcx;
-        let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all());
+        let callee_ty = instance.ty(tcx, self.typing_env());
 
-        let (def_id, substs) = match *callee_ty.kind() {
-            ty::FnDef(def_id, substs) => (def_id, substs),
-            _ => bug!("expected fn item type, found {}", callee_ty),
+        let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else {
+            bug!("expected fn item type, found {}", callee_ty);
         };
 
         let sig = callee_ty.fn_sig(tcx);
-        let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig);
+        let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig);
         let arg_tys = sig.inputs();
         let ret_ty = sig.output();
         let name = tcx.item_name(def_id);
@@ -184,9 +180,7 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
 
         trace!(
             "Beginning intrinsic call: `{:?}`, args: `{:?}`, ret: `{:?}`",
-            name,
-            arg_tys,
-            ret_ty
+            name, arg_tys, ret_ty
         );
 
         let llret_ty = self.layout_of(ret_ty).llvm_type(self);
@@ -194,55 +188,75 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
 
         let simple = get_simple_intrinsic(self, name);
         let llval = match name {
-            _ if simple.is_some() => self.call(
-                self.type_i1(),
-                simple.unwrap(),
-                &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
-                None,
-            ),
-            sym::likely => {
-                let expect = self.get_intrinsic("llvm.expect.i1");
+            _ if simple.is_some() => {
+                let (simple_ty, simple_fn) = simple.unwrap();
                 self.call(
-                    self.type_i1(),
-                    expect,
-                    &[args[0].immediate(), self.const_bool(true)],
+                    simple_ty,
                     None,
-                )
-            }
-            sym::unlikely => {
-                let expect = self.get_intrinsic("llvm.expect.i1");
-                self.call(
-                    self.type_i1(),
-                    expect,
-                    &[args[0].immediate(), self.const_bool(false)],
                     None,
+                    simple_fn,
+                    &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
+                    None,
+                    Some(instance),
                 )
             }
+            sym::is_val_statically_known => {
+                // LLVM 7 does not support this intrinsic, so always assume false.
+                self.const_bool(false)
+            }
+            sym::select_unpredictable => {
+                // This should set MD_unpredictable on the select instruction, but
+                // nvvm ignores it, so just use a normal select.
+                let cond = args[0].immediate();
+                assert_eq!(args[1].layout, args[2].layout);
+                match (args[1].val, args[2].val) {
+                    (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
+                        assert!(true_val.llextra.is_none());
+                        assert!(false_val.llextra.is_none());
+                        assert_eq!(true_val.align, false_val.align);
+                        let ptr = self.select(cond, true_val.llval, false_val.llval);
+                        let selected =
+                            OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
+                        selected.store(self, result);
+                        return Ok(());
+                    }
+                    (OperandValue::Immediate(_), OperandValue::Immediate(_))
+                    | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
+                        let true_val = args[1].immediate_or_packed_pair(self);
+                        let false_val = args[2].immediate_or_packed_pair(self);
+                        self.select(cond, true_val, false_val)
+                    }
+                    (OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()),
+                    _ => span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
+                }
+            }
+            sym::likely => self.call_intrinsic(
+                "llvm.expect.i1",
+                &[args[0].immediate(), self.const_bool(true)],
+            ),
+            sym::unlikely => self.call_intrinsic(
+                "llvm.expect.i1",
+                &[args[0].immediate(), self.const_bool(false)],
+            ),
             kw::Try => {
                 let try_func = args[0].immediate();
                 let data = args[1].immediate();
 
-                self.call(self.type_i1(), try_func, &[data], None);
+                self.call(self.type_i1(), None, None, try_func, &[data], None, None);
                 let ret_align = self.data_layout().i32_align.abi;
                 self.store(self.const_i32(0), llresult, ret_align)
             }
             sym::breakpoint => {
                 // debugtrap is not supported
-                return;
+                return Ok(());
             }
             sym::va_copy => {
-                let intrinsic = self.cx().get_intrinsic("llvm.va_copy");
-                self.call(
-                    self.type_i1(),
-                    intrinsic,
-                    &[args[0].immediate(), args[1].immediate()],
-                    None,
-                )
+                self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()])
             }
             sym::va_arg => {
-                match fn_abi.ret.layout.abi {
-                    abi::Abi::Scalar(ref scalar) => {
-                        match scalar.value {
+                match fn_abi.ret.layout.backend_repr {
+                    abi::BackendRepr::Scalar(scalar) => {
+                        match scalar.primitive() {
                             Primitive::Int(..) => {
                                 if self.cx().size_of(ret_ty).bytes() < 4 {
                                     // `va_arg` should not be called on a integer type
@@ -261,21 +275,29 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                                     )
                                 }
                             }
-                            Primitive::F64 | Primitive::Pointer => self.va_arg(
+                            Primitive::Float(Float::F16) => {
+                                bug!("the va_arg intrinsic does not work with `f16`")
+                            }
+                            Primitive::Float(Float::F64) | Primitive::Pointer(_) => self.va_arg(
                                 args[0].immediate(),
                                 self.cx.layout_of(ret_ty).llvm_type(self.cx),
                             ),
                             // `va_arg` should never be used with the return type f32.
-                            Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"),
+                            Primitive::Float(Float::F32) => {
+                                bug!("the va_arg intrinsic does not work with `f32`")
+                            }
+                            Primitive::Float(Float::F128) => {
+                                bug!("the va_arg intrinsic does not work with `f128`")
+                            }
                         }
                     }
                     _ => bug!("the va_arg intrinsic does not work with non-scalar types"),
                 }
             }
             sym::volatile_load | sym::unaligned_volatile_load => {
-                let tp_ty = substs.type_at(0);
+                let tp_ty = fn_args.type_at(0);
                 let mut ptr = args[0].immediate();
-                if let PassMode::Cast(ty) = fn_abi.ret.mode {
+                if let PassMode::Cast { cast: ty, .. } = &fn_abi.ret.mode {
                     ptr = self.pointercast(ptr, self.type_ptr_to(ty.llvm_type(self)));
                 }
                 let load = self.volatile_load(self.type_i1(), ptr);
@@ -287,23 +309,25 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                 unsafe {
                     llvm::LLVMSetAlignment(load, align);
                 }
-                self.to_immediate(load, self.layout_of(tp_ty))
+                if !result.layout.is_zst() {
+                    self.store_to_place(load, result.val);
+                }
+                return Ok(());
             }
             sym::volatile_store => {
                 let dst = args[0].deref(self.cx());
                 args[1].val.volatile_store(self, dst);
-                return;
+                return Ok(());
             }
             sym::unaligned_volatile_store => {
                 let dst = args[0].deref(self.cx());
                 args[1].val.unaligned_volatile_store(self, dst);
-                return;
+                return Ok(());
             }
             sym::prefetch_read_data
             | sym::prefetch_write_data
             | sym::prefetch_read_instruction
             | sym::prefetch_write_instruction => {
-                let expect = self.get_intrinsic("llvm.prefetch");
                 let (rw, cache_type) = match name {
                     sym::prefetch_read_data => (0, 1),
                     sym::prefetch_write_data => (1, 1),
@@ -311,18 +335,47 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     sym::prefetch_write_instruction => (1, 0),
                     _ => bug!(),
                 };
-                self.call(
-                    self.type_i1(),
-                    expect,
+                self.call_intrinsic(
+                    "llvm.prefetch",
                     &[
                         args[0].immediate(),
                         self.const_i32(rw),
                         args[1].immediate(),
                         self.const_i32(cache_type),
                     ],
-                    None,
                 )
             }
+            sym::carrying_mul_add => {
+                let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
+
+                let wide_llty = self.type_ix(size.bits() * 2);
+                let args = args.as_array().unwrap();
+                let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
+
+                let wide = if signed {
+                    let prod = self.unchecked_smul(a, b);
+                    let acc = self.unchecked_sadd(prod, c);
+                    self.unchecked_sadd(acc, d)
+                } else {
+                    let prod = self.unchecked_umul(a, b);
+                    let acc = self.unchecked_uadd(prod, c);
+                    self.unchecked_uadd(acc, d)
+                };
+
+                let narrow_llty = self.type_ix(size.bits());
+                let low = self.trunc(wide, narrow_llty);
+                let bits_const = self.const_uint(wide_llty, size.bits());
+                // No need for ashr when signed; LLVM changes it to lshr anyway.
+                let high = self.lshr(wide, bits_const);
+                // FIXME: could be `trunc nuw`, even for signed.
+                let high = self.trunc(high, narrow_llty);
+
+                let pair_llty = self.type_struct(&[narrow_llty, narrow_llty], false);
+                let pair = self.const_poison(pair_llty);
+                let pair = self.insert_value(pair, low, 0);
+                let pair = self.insert_value(pair, high, 1);
+                pair
+            }
             sym::ctlz
             | sym::ctlz_nonzero
             | sym::cttz
@@ -335,20 +388,13 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             | sym::saturating_add
             | sym::saturating_sub => {
                 let ty = arg_tys[0];
-                let (width, signed) = if let Some(res) = int_type_width_signed(ty, self.cx) {
-                    res
-                } else {
-                    span_invalid_monomorphization_error(
-                        tcx.sess,
-                        span,
-                        &format!(
-                            "invalid monomorphization of `{}` intrinsic: \
-                                  expected basic integer type, found `{}`",
-                            name, ty
-                        ),
-                    );
-                    return;
-                };
+                if !ty.is_integral() {
+                    tcx.dcx()
+                        .emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
+                    return Ok(());
+                }
+                let (size, signed) = ty.int_size_and_signed(self.tcx);
+                let width = size.bits();
                 if name == sym::saturating_add || name == sym::saturating_sub {
                     saturating_intrinsic_impl(
                         self,
@@ -363,38 +409,31 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     match name {
                         sym::ctlz | sym::cttz => {
                             let y = self.const_bool(false);
-                            let llfn = self.get_intrinsic(&format!("llvm.{}.i{}", name, width));
-                            self.call(self.type_i1(), llfn, &[args[0].immediate(), y], None)
+                            let llvm_name = format!("llvm.{}.i{}", name, width);
+                            self.call_intrinsic(&llvm_name, &[args[0].immediate(), y])
                         }
                         sym::ctlz_nonzero | sym::cttz_nonzero => {
                             let y = self.const_bool(true);
-                            let llvm_name = &format!("llvm.{}.i{}", &name_str[..4], width);
-                            let llfn = self.get_intrinsic(llvm_name);
-                            self.call(self.type_i1(), llfn, &[args[0].immediate(), y], None)
+                            let llvm_name = format!("llvm.{}.i{}", &name_str[..4], width);
+                            self.call_intrinsic(&llvm_name, &[args[0].immediate(), y])
                         }
-                        sym::ctpop => self.call(
-                            self.type_i1(),
-                            self.get_intrinsic(&format!("llvm.ctpop.i{}", width)),
+                        sym::ctpop => self.call_intrinsic(
+                            &format!("llvm.ctpop.i{}", width),
                             &[args[0].immediate()],
-                            None,
                         ),
                         sym::bswap => {
                             if width == 8 {
                                 args[0].immediate() // byte swap a u8/i8 is just a no-op
                             } else {
-                                self.call(
-                                    self.type_i1(),
-                                    self.get_intrinsic(&format!("llvm.bswap.i{}", width)),
+                                self.call_intrinsic(
+                                    &format!("llvm.bswap.i{}", width),
                                     &[args[0].immediate()],
-                                    None,
                                 )
                             }
                         }
-                        sym::bitreverse => self.call(
-                            self.type_i1(),
-                            self.get_intrinsic(&format!("llvm.bitreverse.i{}", width)),
+                        sym::bitreverse => self.call_intrinsic(
+                            &format!("llvm.bitreverse.i{}", width),
                             &[args[0].immediate()],
-                            None,
                         ),
                         sym::rotate_left | sym::rotate_right => {
                             let is_left = name == sym::rotate_left;
@@ -403,8 +442,12 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                             // rotate = funnel shift with first two args the same
                             let llvm_name =
                                 &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width);
-                            let llfn = self.get_intrinsic(llvm_name);
-                            self.call(self.type_i1(), llfn, &[val, val, raw_shift], None)
+
+                            // llvm expects shift to be the same type as the values, but rust
+                            // always uses `u32`.
+                            let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
+
+                            self.call_intrinsic(llvm_name, &[val, val, raw_shift])
                         }
                         sym::saturating_add | sym::saturating_sub => {
                             let is_add = name == sym::saturating_add;
@@ -416,22 +459,21 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                                 if is_add { "add" } else { "sub" },
                                 width
                             );
-                            let llfn = self.get_intrinsic(llvm_name);
-                            self.call(self.type_i1(), llfn, &[lhs, rhs], None)
+                            self.call_intrinsic(&llvm_name, &[lhs, rhs])
                         }
                         _ => unreachable!(),
                     }
                 }
             }
             sym::raw_eq => {
-                use abi::Abi::*;
+                use abi::BackendRepr::*;
                 use rustc_codegen_ssa::common::IntPredicate;
-                let tp_ty = substs.type_at(0);
+                let tp_ty = fn_args.type_at(0);
                 let layout = self.layout_of(tp_ty).layout;
-                let use_integer_compare = match layout.abi {
+                let use_integer_compare = match layout.backend_repr() {
                     Scalar(_) | ScalarPair(_, _) => true,
-                    Uninhabited | Vector { .. } => false,
-                    Aggregate { .. } => {
+                    Vector { .. } => false,
+                    Memory { .. } => {
                         // For rusty ABIs, small aggregates are actually passed
                         // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
                         // so we re-use that same threshold here.
@@ -456,66 +498,108 @@ impl<'a, 'll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     let a_ptr = self.bitcast(a, i8p_ty);
                     let b_ptr = self.bitcast(b, i8p_ty);
                     let n = self.const_usize(layout.size.bytes());
-                    let intrinsic = self.get_intrinsic("memcmp");
-                    let cmp = self.call(self.type_i1(), intrinsic, &[a_ptr, b_ptr, n], None);
+                    let cmp = self.call_intrinsic("memcmp", &[a_ptr, b_ptr, n]);
                     self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0))
                 }
             }
+            sym::compare_bytes => self.call_intrinsic(
+                "memcmp",
+                &[
+                    args[0].immediate(),
+                    args[1].immediate(),
+                    args[2].immediate(),
+                ],
+            ),
+
+            sym::black_box => {
+                args[0].val.store(self, result);
+                let result_val_span = [result.val.llval];
+                // We need to "use" the argument in some way LLVM can't introspect, and on
+                // targets that support it we can typically leverage inline assembly to do
+                // this. LLVM's interpretation of inline assembly is that it's, well, a black
+                // box. This isn't the greatest implementation since it probably deoptimizes
+                // more than we want, but it's so far good enough.
+                //
+                // For zero-sized types, the location pointed to by the result may be
+                // uninitialized. Do not "use" the result in this case; instead just clobber
+                // the memory.
+                let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
+                    ("~{memory}", &[])
+                } else {
+                    ("r,~{memory}", &result_val_span)
+                };
+                crate::asm::inline_asm_call(
+                    self,
+                    "",
+                    constraint,
+                    inputs,
+                    self.type_void(),
+                    true,
+                    false,
+                    llvm::AsmDialect::Att,
+                    &[span],
+                )
+                .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
+
+                // We have copied the value to `result` already.
+                return Ok(());
+            }
             // is this even supported by nvvm? i did not find a definitive answer
             _ if name_str.starts_with("simd_") => todo!("simd intrinsics"),
             _ => bug!("unknown intrinsic '{}'", name),
         };
         trace!("Finish intrinsic call: `{:?}`", llval);
         if !fn_abi.ret.is_ignore() {
-            if let PassMode::Cast(ty) = fn_abi.ret.mode {
-                let ptr_llty = self.type_ptr_to(ty.llvm_type(self));
-                let ptr = self.pointercast(result.llval, ptr_llty);
-                self.store(llval, ptr, result.align);
+            if let PassMode::Cast { cast, .. } = &fn_abi.ret.mode {
+                let ptr_llty = self.type_ptr_to(cast.llvm_type(self));
+                let ptr = self.pointercast(result.val.llval, ptr_llty);
+                self.store(llval, ptr, result.val.align);
             } else {
                 OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
                     .val
                     .store(self, result);
             }
         }
+        Ok(())
     }
 
     fn abort(&mut self) {
         trace!("Generate abort call");
-        let fnname = self.get_intrinsic("llvm.trap");
-        self.call(self.type_i1(), fnname, &[], None);
+        self.call_intrinsic("llvm.trap", &[]);
     }
 
-    fn assume(&mut self, val: Self::Value) {
+    fn assume(&mut self, val: &'ll Value) {
         trace!("Generate assume call with `{:?}`", val);
-        let assume_intrinsic = self.get_intrinsic("llvm.assume");
-        self.call(self.type_i1(), assume_intrinsic, &[val], None);
+        self.call_intrinsic("llvm.assume", &[val]);
     }
 
-    fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
+    fn expect(&mut self, cond: &'ll Value, expected: bool) -> &'ll Value {
         trace!("Generate expect call with `{:?}`, {}", cond, expected);
-        let expect = self.get_intrinsic("llvm.expect.i1");
-        self.call(
-            self.type_i1(),
-            expect,
-            &[cond, self.const_bool(expected)],
-            None,
-        )
+        self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)])
     }
 
-    fn type_test(&mut self, _pointer: Self::Value, _typeid: Self::Value) -> Self::Value {
+    fn type_test(&mut self, _pointer: &'ll Value, _typeid: &'ll Metadata) -> &'ll Value {
+        // LLVM CFI doesnt make sense on the GPU
+        self.const_i32(0)
+    }
+
+    fn type_checked_load(
+        &mut self,
+        _llvtable: Self::Value,
+        _vtable_byte_offset: u64,
+        _typeid: Self::Metadata,
+    ) -> Self::Value {
         // LLVM CFI doesnt make sense on the GPU
         self.const_i32(0)
     }
 
-    fn va_start(&mut self, va_list: &'ll Value) -> Self::Value {
+    fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
         trace!("Generate va_start `{:?}`", va_list);
-        let intrinsic = self.cx().get_intrinsic("llvm.va_start");
-        self.call(self.type_i1(), intrinsic, &[va_list], None)
+        self.call_intrinsic("llvm.va.start", &[va_list])
     }
 
-    fn va_end(&mut self, va_list: &'ll Value) -> Self::Value {
+    fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
         trace!("Generate va_end call `{:?}`", va_list);
-        let intrinsic = self.cx().get_intrinsic("llvm.va_end");
-        self.call(self.type_i1(), intrinsic, &[va_list], None)
+        self.call_intrinsic("llvm.va_end", &[va_list])
     }
 }
diff --git a/crates/rustc_codegen_nvvm/src/lib.rs b/crates/rustc_codegen_nvvm/src/lib.rs
index b224e84b..e1ba6688 100644
--- a/crates/rustc_codegen_nvvm/src/lib.rs
+++ b/crates/rustc_codegen_nvvm/src/lib.rs
@@ -3,27 +3,33 @@
 // make our lives a lot easier for llvm ffi with this. And since rustc's core infra
 // relies on it its almost guaranteed to not be removed/broken
 #![feature(extern_types)]
-#![feature(backtrace)]
+#![feature(hash_raw_entry)]
+#![feature(let_chains)]
+#![feature(slice_as_array)]
 
+extern crate rustc_abi;
 extern crate rustc_arena;
 extern crate rustc_ast;
-extern crate rustc_attr;
-extern crate rustc_codegen_llvm;
+extern crate rustc_attr_parsing;
 extern crate rustc_codegen_ssa;
 extern crate rustc_data_structures;
 extern crate rustc_driver;
 extern crate rustc_errors;
 extern crate rustc_fs_util;
 extern crate rustc_hash;
+extern crate rustc_hashes;
 extern crate rustc_hir;
 extern crate rustc_index;
 extern crate rustc_interface;
+extern crate rustc_macros;
 extern crate rustc_metadata;
 extern crate rustc_middle;
 extern crate rustc_query_system;
 extern crate rustc_session;
 extern crate rustc_span;
+extern crate rustc_symbol_mangling;
 extern crate rustc_target;
+extern crate rustc_type_ir;
 
 mod abi;
 mod allocator;
@@ -31,6 +37,7 @@ mod asm;
 mod attributes;
 mod back;
 mod builder;
+mod common;
 mod const_ty;
 mod consts;
 mod context;
@@ -51,56 +58,85 @@ mod ty;
 use abi::readjust_fn_abi;
 use back::target_machine_factory;
 use lto::ThinBuffer;
+use rustc_ast::expand::allocator::AllocatorKind;
+use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
 use rustc_codegen_ssa::{
+    CodegenResults, CompiledModule, ModuleCodegen,
     back::{
         lto::{LtoModuleCodegen, SerializedModule, ThinModule},
-        write::{CodegenContext, FatLTOInput, ModuleConfig, OngoingCodegen},
+        write::{CodegenContext, FatLtoInput, ModuleConfig, OngoingCodegen},
     },
     traits::{CodegenBackend, ExtraBackendMethods, WriteBackendMethods},
-    CodegenResults, CompiledModule, ModuleCodegen,
 };
-use rustc_errors::{ErrorReported, FatalError, Handler};
-use rustc_hash::FxHashMap;
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_errors::{DiagCtxtHandle, FatalError};
 use rustc_metadata::EncodedMetadata;
-use rustc_middle::ty::query;
+use rustc_metadata::creader::MetadataLoaderDyn;
+use rustc_middle::util::Providers;
 use rustc_middle::{
     dep_graph::{WorkProduct, WorkProductId},
     ty::TyCtxt,
 };
-use rustc_session::{cstore::MetadataLoaderDyn, Session};
+use rustc_session::{
+    Session,
+    config::{self, OutputFilenames},
+};
 use tracing::debug;
 
 use std::ffi::CString;
 
 // codegen dylib entrypoint
-#[no_mangle]
+#[unsafe(no_mangle)]
 pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
-    let _ = std::panic::take_hook();
-    Box::new(NvvmCodegenBackend::default())
+    rustc_driver::install_ice_hook(
+        "https://github.com/Rust-GPU/Rust-CUDA/issues/new",
+        |handler| {
+            handler.handle().note(concat!(
+                "`rust-cuda` version `",
+                env!("CARGO_PKG_VERSION"),
+                "`"
+            ));
+        },
+    );
+    Box::new(NvvmCodegenBackend)
 }
 
-#[derive(Default, Clone)]
-pub struct NvvmCodegenBackend(());
+#[derive(Clone)]
+pub struct NvvmCodegenBackend;
 
 unsafe impl Send for NvvmCodegenBackend {}
 unsafe impl Sync for NvvmCodegenBackend {}
 
 impl CodegenBackend for NvvmCodegenBackend {
+    fn locale_resource(&self) -> &'static str {
+        ""
+    }
+
     fn init(&self, sess: &Session) {
         let filter = tracing_subscriber::EnvFilter::from_env("NVVM_LOG");
         let subscriber = tracing_subscriber::fmt()
             .with_env_filter(filter)
+            .without_time()
+            .with_ansi(false)
             .compact()
             .finish();
 
         tracing::subscriber::set_global_default(subscriber).expect("no default subscriber");
         init::init(sess);
     }
+
+    // FIXME If we can use the default metadata loader in the LLVM backend
+    // we can remove this and use the default provided impl instead.
     fn metadata_loader(&self) -> Box<MetadataLoaderDyn> {
         Box::new(link::NvvmMetadataLoader)
     }
 
-    fn provide(&self, providers: &mut query::Providers) {
+    fn provide(&self, providers: &mut Providers) {
+        // FIXME(eddyb) this is currently only passed back to us, specifically
+        // into `target_machine_factory` (which is a noop), but it might make
+        // sense to move some of the target feature parsing into here.
+        providers.global_backend_features = |_tcx, ()| vec![];
+
         providers.fn_abi_of_fn_ptr = |tcx, key| {
             let result = (rustc_interface::DEFAULT_QUERY_PROVIDERS.fn_abi_of_fn_ptr)(tcx, key);
             Ok(readjust_fn_abi(tcx, result?))
@@ -110,7 +146,6 @@ impl CodegenBackend for NvvmCodegenBackend {
             Ok(readjust_fn_abi(tcx, result?))
         };
     }
-    fn provide_extern(&self, _providers: &mut query::ExternProviders) {}
 
     fn codegen_crate(
         &self,
@@ -120,9 +155,14 @@ impl CodegenBackend for NvvmCodegenBackend {
     ) -> Box<dyn std::any::Any> {
         debug!("Codegen crate");
         Box::new(rustc_codegen_ssa::base::codegen_crate(
-            NvvmCodegenBackend::default(),
+            Self,
             tcx,
-            String::new(),
+            tcx.sess
+                .opts
+                .cg
+                .target_cpu
+                .clone()
+                .unwrap_or_else(|| tcx.sess.target.cpu.to_string()),
             metadata,
             need_metadata_module,
         ))
@@ -132,45 +172,45 @@ impl CodegenBackend for NvvmCodegenBackend {
         &self,
         ongoing_codegen: Box<dyn std::any::Any>,
         sess: &Session,
-    ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorReported> {
+        _outputs: &OutputFilenames,
+    ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
         debug!("Join codegen");
         let (codegen_results, work_products) = ongoing_codegen
             .downcast::<OngoingCodegen<Self>>()
             .expect("Expected OngoingCodegen, found Box<Any>")
             .join(sess);
 
-        sess.compile_status()?;
+        // sess.compile_status();
 
-        Ok((codegen_results, work_products))
+        (codegen_results, work_products)
     }
 
     fn link(
         &self,
         sess: &rustc_session::Session,
         codegen_results: rustc_codegen_ssa::CodegenResults,
-        outputs: &rustc_session::config::OutputFilenames,
-    ) -> Result<(), rustc_errors::ErrorReported> {
+        outputs: &config::OutputFilenames,
+    ) {
         link::link(
             sess,
             &codegen_results,
             outputs,
             &codegen_results.crate_info.local_crate_name.as_str(),
         );
-        Ok(())
     }
 }
 
 impl WriteBackendMethods for NvvmCodegenBackend {
     type Module = LlvmMod;
     type ModuleBuffer = lto::ModuleBuffer;
-    type Context = llvm::Context;
     type TargetMachine = &'static mut llvm::TargetMachine;
+    type TargetMachineError = String;
     type ThinData = ();
     type ThinBuffer = ThinBuffer;
 
     fn run_link(
         _cgcx: &CodegenContext<Self>,
-        _diag_handler: &Handler,
+        _diag_handler: DiagCtxtHandle<'_>,
         _modules: Vec<ModuleCodegen<Self::Module>>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
         // TODO(Rdambrosio016):
@@ -183,7 +223,7 @@ impl WriteBackendMethods for NvvmCodegenBackend {
 
     fn run_fat_lto(
         _: &CodegenContext<Self>,
-        _: Vec<FatLTOInput<Self>>,
+        _: Vec<FatLtoInput<Self>>,
         _: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
     ) -> Result<LtoModuleCodegen<Self>, FatalError> {
         todo!()
@@ -201,32 +241,39 @@ impl WriteBackendMethods for NvvmCodegenBackend {
         // Not applicable, nvvm doesnt expose pass timing info, maybe we could print llvm pass stuff here.
     }
 
+    fn print_statistics(&self) {
+        // Not applicable, nvvm doesnt expose pass timing info, maybe we could print llvm pass stuff here.
+    }
+
     unsafe fn optimize(
         cgcx: &CodegenContext<Self>,
-        diag_handler: &Handler,
-        module: &ModuleCodegen<Self::Module>,
+        diag_handler: DiagCtxtHandle<'_>,
+        module: &mut ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<(), FatalError> {
-        back::optimize(cgcx, diag_handler, module, config)
+        unsafe { back::optimize(cgcx, diag_handler, module, config) }
     }
 
     unsafe fn optimize_thin(
         cgcx: &CodegenContext<Self>,
-        thin_module: &mut ThinModule<Self>,
+        thin_module: ThinModule<Self>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
-        lto::optimize_thin(cgcx, thin_module)
+        unsafe { lto::optimize_thin(cgcx, thin_module) }
     }
 
     unsafe fn codegen(
         cgcx: &CodegenContext<Self>,
-        diag_handler: &Handler,
+        diag_handler: DiagCtxtHandle<'_>,
         module: ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<CompiledModule, FatalError> {
-        back::codegen(cgcx, diag_handler, module, config)
+        unsafe { back::codegen(cgcx, diag_handler, module, config) }
     }
 
-    fn prepare_thin(module: ModuleCodegen<Self::Module>) -> (String, Self::ThinBuffer) {
+    fn prepare_thin(
+        module: ModuleCodegen<Self::Module>,
+        _want_summary: bool,
+    ) -> (String, Self::ThinBuffer) {
         debug!("Prepare thin");
         unsafe {
             (
@@ -246,39 +293,42 @@ impl WriteBackendMethods for NvvmCodegenBackend {
         }
     }
 
-    fn run_lto_pass_manager(
-        _: &CodegenContext<Self>,
-        _: &ModuleCodegen<Self::Module>,
-        _: &ModuleConfig,
-        _: bool,
+    fn optimize_fat(
+        _cgcx: &CodegenContext<Self>,
+        _llmod: &mut ModuleCodegen<Self::Module>,
     ) -> Result<(), FatalError> {
         todo!()
     }
-}
 
-impl ExtraBackendMethods for NvvmCodegenBackend {
-    fn new_metadata(&self, _sess: TyCtxt<'_>, mod_name: &str) -> Self::Module {
-        LlvmMod::new(mod_name)
-    }
-
-    fn write_compressed_metadata<'tcx>(
-        &self,
-        _tcx: TyCtxt<'tcx>,
-        _metadata: &EncodedMetadata,
-        _llvm_module: &mut Self::Module,
-    ) {
+    fn autodiff(
+        _cgcx: &CodegenContext<Self>,
+        _module: &ModuleCodegen<Self::Module>,
+        _diff_fncs: Vec<AutoDiffItem>,
+        _config: &ModuleConfig,
+    ) -> Result<(), FatalError> {
         todo!()
     }
+}
 
+impl ExtraBackendMethods for NvvmCodegenBackend {
     fn codegen_allocator<'tcx>(
         &self,
         tcx: TyCtxt<'tcx>,
-        mods: &mut Self::Module,
-        _module_name: &str,
-        kind: rustc_ast::expand::allocator::AllocatorKind,
-        has_alloc_error_handler: bool,
-    ) {
-        unsafe { allocator::codegen(tcx, mods, kind, has_alloc_error_handler) }
+        module_name: &str,
+        kind: AllocatorKind,
+        alloc_error_handler_kind: AllocatorKind,
+    ) -> LlvmMod {
+        let mut module_llvm = LlvmMod::new(module_name);
+        unsafe {
+            allocator::codegen(
+                tcx,
+                &mut module_llvm,
+                module_name,
+                kind,
+                alloc_error_handler_kind,
+            );
+        }
+        module_llvm
     }
 
     fn compile_codegen_unit(
@@ -292,18 +342,11 @@ impl ExtraBackendMethods for NvvmCodegenBackend {
     fn target_machine_factory(
         &self,
         sess: &Session,
-        opt_level: rustc_session::config::OptLevel,
+        opt_level: config::OptLevel,
+        _target_features: &[String],
     ) -> rustc_codegen_ssa::back::write::TargetMachineFactoryFn<Self> {
         target_machine_factory(sess, opt_level)
     }
-
-    fn target_cpu<'b>(&self, _sess: &'b Session) -> &'b str {
-        todo!()
-    }
-
-    fn tune_cpu<'b>(&self, _sess: &'b Session) -> Option<&'b str> {
-        todo!()
-    }
 }
 
 /// Create the LLVM module for the rest of the compilation, this houses
@@ -315,13 +358,13 @@ pub(crate) unsafe fn create_module<'ll>(
 ) -> &'ll llvm::Module {
     debug!("Creating llvm module with name `{}`", mod_name);
     let mod_name = CString::new(mod_name).expect("nul in module name");
-    let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx);
+    let llmod = unsafe { llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx) };
 
     let data_layout = CString::new(target::DATA_LAYOUT).unwrap();
-    llvm::LLVMSetDataLayout(llmod, data_layout.as_ptr());
+    unsafe { llvm::LLVMSetDataLayout(llmod, data_layout.as_ptr()) };
 
     let target = CString::new(target::TARGET_TRIPLE).unwrap();
-    llvm::LLVMSetTarget(llmod, target.as_ptr());
+    unsafe { llvm::LLVMSetTarget(llmod, target.as_ptr()) };
 
     llmod
 }
diff --git a/crates/rustc_codegen_nvvm/src/link.rs b/crates/rustc_codegen_nvvm/src/link.rs
index 0d5e4840..5ca9d12a 100644
--- a/crates/rustc_codegen_nvvm/src/link.rs
+++ b/crates/rustc_codegen_nvvm/src/link.rs
@@ -1,20 +1,24 @@
+use object::{Object, ObjectSection};
+use rustc_ast::CRATE_NODE_ID;
 use rustc_codegen_ssa::CodegenResults;
 use rustc_codegen_ssa::CompiledModule;
 use rustc_codegen_ssa::NativeLib;
-use rustc_data_structures::owning_ref::OwningRef;
-use rustc_data_structures::rustc_erase_owner;
-use rustc_data_structures::sync::MetadataRef;
+use rustc_data_structures::memmap::Mmap;
+use rustc_data_structures::owned_slice::{OwnedSlice, slice_owned, try_slice_owned};
 use rustc_hash::FxHashSet;
+use rustc_metadata::creader::MetadataLoader;
+use rustc_middle::bug;
 use rustc_middle::middle::dependency_format::Linkage;
-use rustc_session::cstore::MetadataLoader;
 use rustc_session::output::out_filename;
 use rustc_session::{
+    Session,
     config::{CrateType, OutputFilenames, OutputType},
     output::check_file_is_writeable,
     utils::NativeLibKind,
-    Session,
 };
+use rustc_span::Symbol;
 use rustc_target::spec::Target;
+use std::ops::Deref;
 use std::{
     ffi::OsStr,
     fs::File,
@@ -24,35 +28,48 @@ use std::{
 use tar::{Archive, Builder, Header};
 use tracing::{debug, trace};
 
-use crate::context::CodegenArgs;
 use crate::LlvmMod;
+use crate::context::CodegenArgs;
 
 pub(crate) struct NvvmMetadataLoader;
 
+fn load_metadata_with(
+    path: &Path,
+    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
+) -> Result<OwnedSlice, String> {
+    let file =
+        File::open(path).map_err(|e| format!("failed to open file '{}': {}", path.display(), e))?;
+
+    unsafe { Mmap::map(file) }
+        .map_err(|e| format!("failed to mmap file '{}': {}", path.display(), e))
+        .and_then(|mmap| try_slice_owned(mmap, |mmap| f(mmap)))
+}
+
 impl MetadataLoader for NvvmMetadataLoader {
-    fn get_rlib_metadata(&self, _target: &Target, filename: &Path) -> Result<MetadataRef, String> {
-        trace!("Retrieving rlib metadata for `{:?}`", filename);
-        read_metadata(filename)
+    fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<OwnedSlice, String> {
+        trace!("Retrieving rlib metadata for `{:?}`", path);
+        read_metadata(path)
     }
 
-    fn get_dylib_metadata(&self, target: &Target, filename: &Path) -> Result<MetadataRef, String> {
-        trace!("Retrieving dylib metadata for `{:?}`", filename);
+    fn get_dylib_metadata(&self, target: &Target, path: &Path) -> Result<OwnedSlice, String> {
+        debug!("getting rlib metadata for {}", path.display());
         // This is required for loading metadata from proc macro crates compiled as dylibs for the host target.
-        rustc_codegen_llvm::LlvmCodegenBackend::new()
-            .metadata_loader()
-            .get_dylib_metadata(target, filename)
+        if target.is_like_aix {
+            bug!("aix dynlibs unsupported");
+        } else {
+            load_metadata_with(path, |data| search_for_section(path, data, ".rustc"))
+        }
     }
 }
 
-fn read_metadata(rlib: &Path) -> Result<MetadataRef, String> {
-    let read_meta = || -> Result<Option<MetadataRef>, io::Error> {
+fn read_metadata(rlib: &Path) -> Result<OwnedSlice, String> {
+    let read_meta = || -> Result<Option<OwnedSlice>, io::Error> {
         for entry in Archive::new(File::open(rlib)?).entries()? {
             let mut entry = entry?;
             if entry.path()? == Path::new(".metadata") {
                 let mut bytes = Vec::new();
                 entry.read_to_end(&mut bytes)?;
-                let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(bytes);
-                return Ok(Some(rustc_erase_owner!(buf.map_owner_box())));
+                return Ok(Some(slice_owned(bytes, Deref::deref)));
             }
         }
         Ok(None)
@@ -65,6 +82,29 @@ fn read_metadata(rlib: &Path) -> Result<MetadataRef, String> {
     }
 }
 
+fn search_for_section<'a>(path: &Path, bytes: &'a [u8], section: &str) -> Result<&'a [u8], String> {
+    let Ok(file) = object::File::parse(bytes) else {
+        // The parse above could fail for odd reasons like corruption, but for
+        // now we just interpret it as this target doesn't support metadata
+        // emission in object files so the entire byte slice itself is probably
+        // a metadata file. Ideally though if necessary we could at least check
+        // the prefix of bytes to see if it's an actual metadata object and if
+        // not forward the error along here.
+        return Ok(bytes);
+    };
+    file.section_by_name(section)
+        .ok_or_else(|| format!("no `{}` section in '{}'", section, path.display()))?
+        .data()
+        .map_err(|e| {
+            format!(
+                "failed to read {} section in '{}': {}",
+                section,
+                path.display(),
+                e
+            )
+        })
+}
+
 pub fn link<'tcx>(
     sess: &'tcx Session,
     codegen_results: &CodegenResults,
@@ -74,8 +114,8 @@ pub fn link<'tcx>(
     debug!("Linking crate `{}`", crate_name);
     // largely inspired by rust-gpu
     let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
-    for &crate_type in sess.crate_types().iter() {
-        if (sess.opts.debugging_opts.no_codegen || !sess.opts.output_types.should_codegen())
+    for &crate_type in sess.opts.crate_types.iter() {
+        if (sess.opts.unstable_opts.no_codegen || !sess.opts.output_types.should_codegen())
             && !output_metadata
             && crate_type == CrateType::Executable
         {
@@ -91,21 +131,23 @@ pub fn link<'tcx>(
         }
 
         if outputs.outputs.should_codegen() {
-            let out_filename = out_filename(sess, crate_type, outputs, crate_name);
+            let out_filename = out_filename(sess, crate_type, outputs, Symbol::intern(crate_name));
+            let out_filename_file_for_writing =
+                out_filename.file_for_writing(outputs, OutputType::Exe, None);
             match crate_type {
                 CrateType::Rlib => {
-                    link_rlib(sess, codegen_results, &out_filename);
+                    link_rlib(sess, codegen_results, &out_filename_file_for_writing);
                 }
                 CrateType::Executable | CrateType::Cdylib | CrateType::Dylib => {
                     let _ = link_exe(
                         &codegen_results.allocator_module,
                         sess,
                         crate_type,
-                        &out_filename,
+                        &out_filename_file_for_writing,
                         codegen_results,
                     );
                 }
-                other => sess.fatal(&format!("Invalid crate type: {:?}", other)),
+                other => sess.dcx().fatal(format!("Invalid crate type: {:?}", other)),
             }
         }
     }
@@ -124,29 +166,19 @@ fn link_rlib(sess: &Session, codegen_results: &CodegenResults, out_filename: &Pa
     }
 
     for lib in codegen_results.crate_info.used_libraries.iter() {
-        match lib.kind {
-            NativeLibKind::Static {
-                bundle: None | Some(true),
-                ..
-            } => {}
-            NativeLibKind::Static {
-                bundle: Some(false),
-                ..
-            }
-            | NativeLibKind::Dylib { .. }
-            | NativeLibKind::Framework { .. }
-            | NativeLibKind::RawDylib
-            | NativeLibKind::Unspecified => continue,
-        }
         // native libraries in cuda doesnt make much sense, extern functions
         // do exist in nvvm for stuff like cuda syscalls and cuda provided functions
         // but including libraries doesnt make sense because nvvm would have to translate
         // the binary directly to ptx. We might want to add some way of linking in
         // ptx files or custom bitcode modules as "libraries" perhaps in the future.
-        if let Some(name) = lib.name {
-            sess.err(&format!(
+        if let NativeLibKind::Static {
+            bundle: None | Some(true),
+            ..
+        } = lib.kind
+        {
+            sess.dcx().err(format!(
                 "Adding native libraries to rlib is not supported in CUDA: {}",
-                name
+                lib.name
             ));
         }
     }
@@ -203,7 +235,8 @@ fn codegen_into_ptx_file(
     rlibs: &[PathBuf],
     out_filename: &Path,
 ) -> io::Result<()> {
-    debug!("Codegenning crate into PTX, allocator: {}, objects:\n{:#?}, rlibs:\n{:#?}, out_filename:\n{:#?}",
+    debug!(
+        "Codegenning crate into PTX, allocator: {}, objects:\n{:#?}, rlibs:\n{:#?}, out_filename:\n{:#?}",
         allocator.is_some(),
         objects,
         rlibs,
@@ -262,7 +295,7 @@ fn codegen_into_ptx_file(
         Ok(bytes) => bytes,
         Err(err) => {
             // TODO(RDambrosio016): maybe include the nvvm log with this fatal error
-            sess.fatal(&err.to_string())
+            sess.dcx().fatal(err.to_string())
         }
     };
 
@@ -271,7 +304,8 @@ fn codegen_into_ptx_file(
 
 fn create_archive(sess: &Session, files: &[&Path], metadata: &[u8], out_filename: &Path) {
     if let Err(err) = try_create_archive(files, metadata, out_filename) {
-        sess.fatal(&format!("Failed to create archive: {}", err));
+        sess.dcx()
+            .fatal(format!("Failed to create archive: {}", err));
     }
 }
 
@@ -307,11 +341,11 @@ fn link_local_crate_native_libs_and_dependent_crate_libs<'a>(
     crate_type: CrateType,
     codegen_results: &CodegenResults,
 ) {
-    if sess.opts.debugging_opts.link_native_libraries {
+    if sess.opts.unstable_opts.link_native_libraries {
         add_local_native_libraries(sess, codegen_results);
     }
     add_upstream_rust_crates(sess, rlibs, codegen_results, crate_type);
-    if sess.opts.debugging_opts.link_native_libraries {
+    if sess.opts.unstable_opts.link_native_libraries {
         add_upstream_native_libraries(sess, codegen_results, crate_type);
     }
 }
@@ -335,17 +369,17 @@ fn add_upstream_rust_crates(
         .crate_info
         .dependency_formats
         .iter()
-        .find(|(ty, _)| *ty == crate_type)
+        .find(|(ty, _)| **ty == crate_type)
         .expect("failed to find crate type in dependency format list");
     let deps = &codegen_results.crate_info.used_crates;
     for cnum in deps.iter() {
         let src = &codegen_results.crate_info.used_crate_source[cnum];
-        match data[cnum.as_usize() - 1] {
+        match data[*cnum] {
             Linkage::NotLinked => {}
             Linkage::Static => rlibs.push(src.rlib.as_ref().unwrap().0.clone()),
             // should we just ignore includedFromDylib?
             Linkage::Dynamic | Linkage::IncludedFromDylib => {
-                sess.fatal("Dynamic Linking is not supported in CUDA")
+                sess.dcx().fatal("Dynamic Linking is not supported in CUDA")
             }
         }
     }
@@ -362,14 +396,15 @@ fn add_upstream_native_libraries(
             if !relevant_lib(sess, lib) {
                 continue;
             }
-            sess.fatal("Native libraries are not supported in CUDA");
+            sess.dcx()
+                .fatal("Native libraries are not supported in CUDA");
         }
     }
 }
 
 fn relevant_lib(sess: &Session, lib: &NativeLib) -> bool {
-    match lib.cfg {
-        Some(ref cfg) => rustc_attr::cfg_matches(cfg, &sess.parse_sess, None),
+    match &lib.cfg {
+        Some(cfg) => rustc_attr_parsing::cfg_matches(cfg, sess, CRATE_NODE_ID, None),
         None => true,
     }
 }
diff --git a/crates/rustc_codegen_nvvm/src/llvm.rs b/crates/rustc_codegen_nvvm/src/llvm.rs
index fe3a4c9e..a0243eed 100644
--- a/crates/rustc_codegen_nvvm/src/llvm.rs
+++ b/crates/rustc_codegen_nvvm/src/llvm.rs
@@ -346,14 +346,14 @@ pub enum AsmDialect {
     Intel,
 }
 
-impl AsmDialect {
-    pub fn from_generic(asm: rustc_ast::LlvmAsmDialect) -> Self {
-        match asm {
-            rustc_ast::LlvmAsmDialect::Att => AsmDialect::Att,
-            rustc_ast::LlvmAsmDialect::Intel => AsmDialect::Intel,
-        }
-    }
-}
+// impl AsmDialect {
+//     pub fn from_generic(asm: rustc_ast::LlvmAsmDialect) -> Self {
+//         match asm {
+//             rustc_ast::LlvmAsmDialect::Att => AsmDialect::Att,
+//             rustc_ast::LlvmAsmDialect::Intel => AsmDialect::Intel,
+//         }
+//     }
+// }
 
 /// LLVMRustDiagnosticKind
 #[derive(Copy, Clone)]
@@ -409,15 +409,15 @@ pub(crate) enum PassKind {
     Module,
 }
 
-/// LLVMRustThinLTOData
-extern "C" {
+// LLVMRustThinLTOData
+unsafe extern "C" {
     pub(crate) type ThinLTOData;
 }
 
 unsafe impl Send for ThinLTOData {}
 
-/// LLVMRustThinLTOBuffer
-extern "C" {
+// LLVMRustThinLTOBuffer
+unsafe extern "C" {
     pub(crate) type ThinLTOBuffer;
 }
 
@@ -431,7 +431,7 @@ pub(crate) struct ThinLTOModule {
     pub len: usize,
 }
 
-extern "C" {
+unsafe extern "C" {
     type Opaque;
 }
 #[repr(C)]
@@ -441,28 +441,28 @@ struct InvariantOpaque<'a> {
 }
 
 // Opaque pointer types
-extern "C" {
+unsafe extern "C" {
     pub(crate) type Module;
 }
-extern "C" {
+unsafe extern "C" {
     pub type Context;
 }
 
 unsafe impl Send for Context {}
 
-extern "C" {
+unsafe extern "C" {
     pub(crate) type Type;
 }
-extern "C" {
+unsafe extern "C" {
     pub(crate) type Value;
 }
-extern "C" {
+unsafe extern "C" {
     pub(crate) type ConstantInt;
 }
-extern "C" {
+unsafe extern "C" {
     pub type Metadata;
 }
-extern "C" {
+unsafe extern "C" {
     pub(crate) type BasicBlock;
 }
 #[repr(C)]
@@ -472,7 +472,7 @@ pub(crate) struct Builder<'a> {
 #[repr(C)]
 pub(crate) struct OperandBundleDef<'a>(InvariantOpaque<'a>);
 
-extern "C" {
+unsafe extern "C" {
     pub(crate) type ModuleBuffer;
 }
 
@@ -480,16 +480,16 @@ unsafe impl Send for ModuleBuffer {}
 
 #[repr(C)]
 pub struct PassManager<'a>(InvariantOpaque<'a>);
-extern "C" {
+unsafe extern "C" {
     pub type PassManagerBuilder;
 }
-extern "C" {
+unsafe extern "C" {
     pub type Pass;
 }
-extern "C" {
+unsafe extern "C" {
     pub type TargetMachine;
 }
-extern "C" {
+unsafe extern "C" {
     pub(crate) type MemoryBuffer;
 }
 
@@ -500,7 +500,6 @@ pub enum ChecksumKind {
     None,
     MD5,
     SHA1,
-    SHA256,
 }
 
 pub mod debuginfo {
@@ -528,10 +527,13 @@ pub mod debuginfo {
     pub type DIEnumerator = DIDescriptor;
     pub type DITemplateTypeParameter = DIDescriptor;
 
-    // These values **must** match with LLVMRustDIFlags!!
     bitflags! {
+        /// Must match the layout of `LLVMDIFlags` in the LLVM-C API.
+        ///
+        /// Each value declared here must also be covered by the static
+        /// assertions in `RustWrapper.cpp` used by `fromRust(LLVMDIFlags)`.
         #[repr(C)]
-        #[derive(Default)]
+        #[derive(Clone, Copy, Default)]
         pub struct DIFlags: u32 {
             const FlagZero                = 0;
             const FlagPrivate             = 1;
@@ -557,6 +559,37 @@ pub mod debuginfo {
             const FlagMainSubprogram      = (1 << 21);
         }
     }
+
+    /// LLVMRustDebugEmissionKind
+    #[derive(Copy, Clone)]
+    #[repr(C)]
+    pub enum DebugEmissionKind {
+        NoDebug,
+        FullDebug,
+        LineTablesOnly,
+    }
+
+    impl DebugEmissionKind {
+        pub(crate) fn from_generic(kind: rustc_session::config::DebugInfo) -> Self {
+            // We should be setting LLVM's emission kind to `LineTablesOnly` if
+            // we are compiling with "limited" debuginfo. However, some of the
+            // existing tools relied on slightly more debuginfo being generated than
+            // would be the case with `LineTablesOnly`, and we did not want to break
+            // these tools in a "drive-by fix", without a good idea or plan about
+            // what limited debuginfo should exactly look like. So for now we are
+            // instead adding a new debuginfo option "line-tables-only" so as to
+            // not break anything and to allow users to have 'limited' debug info.
+            //
+            // See https://github.com/rust-lang/rust/issues/60020 for details.
+            use rustc_session::config::DebugInfo;
+            match kind {
+                // NVVM: Llvm 7 is missing LineDirectivesOnly, so don't emit anything.
+                DebugInfo::None | DebugInfo::LineDirectivesOnly => DebugEmissionKind::NoDebug,
+                DebugInfo::LineTablesOnly => DebugEmissionKind::LineTablesOnly,
+                DebugInfo::Limited | DebugInfo::Full => DebugEmissionKind::FullDebug,
+            }
+        }
+    }
 }
 
 // These functions are kind of a hack for the future. They wrap LLVM 7 rust shim functions
@@ -569,9 +602,11 @@ pub(crate) unsafe fn LLVMRustGetOrInsertFunction<'a>(
     NameLen: usize,
     FunctionTy: &'a Type,
 ) -> &'a Value {
-    let str = std::str::from_utf8_unchecked(std::slice::from_raw_parts(Name.cast(), NameLen));
-    let cstring = CString::new(str).expect("str with nul");
-    __LLVMRustGetOrInsertFunction(M, cstring.as_ptr(), FunctionTy)
+    unsafe {
+        let str = std::str::from_utf8_unchecked(std::slice::from_raw_parts(Name.cast(), NameLen));
+        let cstring = CString::new(str).expect("str with nul");
+        __LLVMRustGetOrInsertFunction(M, cstring.as_ptr(), FunctionTy)
+    }
 }
 
 pub(crate) unsafe fn LLVMRustBuildCall<'a>(
@@ -581,7 +616,7 @@ pub(crate) unsafe fn LLVMRustBuildCall<'a>(
     NumArgs: c_uint,
     Bundle: Option<&OperandBundleDef<'a>>,
 ) -> &'a Value {
-    __LLVMRustBuildCall(B, Fn, Args, NumArgs, Bundle, unnamed())
+    unsafe { __LLVMRustBuildCall(B, Fn, Args, NumArgs, Bundle, unnamed()) }
 }
 
 /// LLVMRustCodeGenOptLevel
@@ -621,7 +656,7 @@ pub enum CodeModel {
     None,
 }
 
-extern "C" {
+unsafe extern "C" {
     #[link_name = "LLVMRustBuildCall"]
     pub(crate) fn __LLVMRustBuildCall<'a>(
         B: &Builder<'a>,
@@ -652,7 +687,7 @@ extern "C" {
 
 // use rustc_codegen_nvvm_macros::trace_ffi_calls;
 // #[trace_ffi_calls]
-extern "C" {
+unsafe extern "C" {
     pub(crate) fn LLVMGetPointerAddressSpace(PointerTy: &Type) -> c_uint;
     pub(crate) fn LLVMBuildAddrSpaceCast<'a>(
         arg1: &Builder<'a>,
@@ -699,7 +734,8 @@ extern "C" {
     ) -> &'a mut MemoryBuffer;
     pub(crate) fn LLVMDisposeMemoryBuffer<'a>(MemBuf: &'a mut MemoryBuffer);
 
-    pub(crate) fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: &'a Value);
+    pub(crate) fn LLVMGetCurrentDebugLocation<'a>(Builder: &Builder<'a>) -> Option<&'a Value>;
+    pub(crate) fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: Option<&'a Value>);
 
     pub(crate) fn LLVMGetModuleContext(M: &Module) -> &Context;
     pub(crate) fn LLVMGetMDKindIDInContext(
@@ -722,21 +758,33 @@ extern "C" {
 
     pub(crate) fn LLVMRustDIBuilderFinalize<'a>(Builder: &DIBuilder<'a>);
 
-    pub(crate) fn LLVMRustDIBuilderCreateCompileUnit<'a>(
+    pub fn LLVMRustDIBuilderCreateCompileUnit<'a>(
         Builder: &DIBuilder<'a>,
         Lang: c_uint,
         File: &'a DIFile,
         Producer: *const c_char,
+        ProducerLen: size_t,
         isOptimized: bool,
         Flags: *const c_char,
         RuntimeVer: c_uint,
         SplitName: *const c_char,
+        SplitNameLen: size_t,
+        kind: DebugEmissionKind,
+        DWOId: u64,
+        SplitDebugInlining: bool,
     ) -> &'a DIDescriptor;
 
     pub(crate) fn LLVMRustDIBuilderCreateFile<'a>(
         Builder: &DIBuilder<'a>,
         Filename: *const c_char,
+        FileNameLen: size_t,
         Directory: *const c_char,
+        DirectoryLen: size_t,
+        CSKind: ChecksumKind,
+        Checksum: *const c_char,
+        ChecksomLen: size_t,
+        Source: *const c_char,
+        SourceLen: size_t,
     ) -> &'a DIFile;
 
     pub(crate) fn LLVMRustDIBuilderCreateSubroutineType<'a>(
@@ -765,10 +813,21 @@ extern "C" {
     pub(crate) fn LLVMRustDIBuilderCreateBasicType<'a>(
         Builder: &DIBuilder<'a>,
         Name: *const c_char,
+        NameLen: size_t,
         SizeInBits: u64,
         Encoding: c_uint,
     ) -> &'a DIBasicType;
 
+    pub fn LLVMRustDIBuilderCreateTypedef<'a>(
+        Builder: &DIBuilder<'a>,
+        Type: &'a DIBasicType,
+        Name: *const c_char,
+        NameLen: size_t,
+        File: &'a DIFile,
+        LineNo: c_uint,
+        Scope: Option<&'a DIScope>,
+    ) -> &'a DIDerivedType;
+
     pub(crate) fn LLVMRustDIBuilderCreatePointerType<'a>(
         Builder: &DIBuilder<'a>,
         PointeeTy: &'a DIType,
@@ -781,6 +840,7 @@ extern "C" {
         Builder: &DIBuilder<'a>,
         Scope: Option<&'a DIDescriptor>,
         Name: *const c_char,
+        NameLen: size_t,
         File: &'a DIFile,
         LineNumber: c_uint,
         SizeInBits: u64,
@@ -791,12 +851,14 @@ extern "C" {
         RunTimeLang: c_uint,
         VTableHolder: Option<&'a DIType>,
         UniqueId: *const c_char,
+        UniqueIdLen: size_t,
     ) -> &'a DICompositeType;
 
     pub(crate) fn LLVMRustDIBuilderCreateMemberType<'a>(
         Builder: &DIBuilder<'a>,
         Scope: &'a DIDescriptor,
         Name: *const c_char,
+        NameLen: size_t,
         File: &'a DIFile,
         LineNo: c_uint,
         SizeInBits: u64,
@@ -810,6 +872,7 @@ extern "C" {
         Builder: &DIBuilder<'a>,
         Scope: &'a DIScope,
         Name: *const c_char,
+        NameLen: size_t,
         File: &'a DIFile,
         LineNumber: c_uint,
         SizeInBits: u64,
@@ -838,7 +901,9 @@ extern "C" {
         Builder: &DIBuilder<'a>,
         Context: Option<&'a DIScope>,
         Name: *const c_char,
+        NameLen: size_t,
         LinkageName: *const c_char,
+        LinkageNameLen: size_t,
         File: &'a DIFile,
         LineNo: c_uint,
         Ty: &'a DIType,
@@ -903,13 +968,16 @@ extern "C" {
     pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>(
         Builder: &DIBuilder<'a>,
         Name: *const c_char,
-        Val: u64,
+        NameLen: size_t,
+        Val: i64,
+        IsUnsigned: bool,
     ) -> &'a DIEnumerator;
 
     pub(crate) fn LLVMRustDIBuilderCreateEnumerationType<'a>(
         Builder: &DIBuilder<'a>,
         Scope: &'a DIScope,
         Name: *const c_char,
+        NameLen: size_t,
         File: &'a DIFile,
         LineNumber: c_uint,
         SizeInBits: u64,
@@ -922,6 +990,7 @@ extern "C" {
         Builder: &DIBuilder<'a>,
         Scope: Option<&'a DIScope>,
         Name: *const c_char,
+        NameLen: size_t,
         File: &'a DIFile,
         LineNumber: c_uint,
         SizeInBits: u64,
@@ -930,12 +999,14 @@ extern "C" {
         Elements: Option<&'a DIArray>,
         RunTimeLang: c_uint,
         UniqueId: *const c_char,
+        UniqueIdLen: size_t,
     ) -> &'a DIType;
 
     pub(crate) fn LLVMRustDIBuilderCreateVariantPart<'a>(
         Builder: &DIBuilder<'a>,
         Scope: &'a DIScope,
         Name: *const c_char,
+        NameLen: size_t,
         File: &'a DIFile,
         LineNo: c_uint,
         SizeInBits: u64,
@@ -944,6 +1015,7 @@ extern "C" {
         Discriminator: Option<&'a DIDerivedType>,
         Elements: &'a DIArray,
         UniqueId: *const c_char,
+        UniqueIdLen: size_t,
     ) -> &'a DIDerivedType;
 
     pub(crate) fn LLVMSetUnnamedAddr<'a>(GlobalVar: &'a Value, UnnamedAddr: Bool);
@@ -952,6 +1024,7 @@ extern "C" {
         Builder: &DIBuilder<'a>,
         Scope: Option<&'a DIScope>,
         Name: *const c_char,
+        NameLen: size_t,
         Ty: &'a DIType,
     ) -> &'a DITemplateTypeParameter;
 
@@ -959,6 +1032,7 @@ extern "C" {
         Builder: &DIBuilder<'a>,
         Scope: Option<&'a DIScope>,
         Name: *const c_char,
+        NameLen: size_t,
     ) -> &'a DINameSpace;
 
     pub(crate) fn LLVMRustDICompositeTypeReplaceArrays<'a>(
@@ -980,8 +1054,10 @@ extern "C" {
         Scope: &'a DIScope,
         InlinedAt: Option<&'a Metadata>,
     ) -> &'a DILocation;
-    pub fn LLVMRustDIBuilderCreateOpDeref() -> i64;
-    pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> i64;
+    pub(crate) fn LLVMRustDILocationCloneWithBaseDiscriminator<'a>(
+        Location: &'a DILocation,
+        BD: c_uint,
+    ) -> Option<&'a DILocation>;
 
     pub(crate) fn LLVMRustRunFunctionPassManager(PM: &PassManager, M: &Module);
     pub(crate) fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool);
@@ -1005,8 +1081,11 @@ extern "C" {
 
     pub(crate) fn LLVMRustCreateTargetMachine<'a>(
         Triple: *const c_char,
+        TripleLen: size_t,
         CPU: *const c_char,
+        CPULen: size_t,
         Features: *const c_char,
+        FeaturesLen: size_t,
         Model: CodeModel,
         Reloc: RelocMode,
         Level: CodeGenOptLevel,
@@ -1024,7 +1103,10 @@ extern "C" {
         M: &'a Module,
     );
     pub(crate) fn LLVMRustPassKind(Pass: &Pass) -> PassKind;
-    pub(crate) fn LLVMRustFindAndCreatePass(Pass: *const c_char) -> Option<&'static mut Pass>;
+    pub(crate) fn LLVMRustFindAndCreatePass(
+        Pass: *const c_char,
+        PassNameLen: size_t,
+    ) -> Option<&'static mut Pass>;
     pub(crate) fn LLVMRustAddPass<'a>(PM: &'a PassManager, Pass: &'static mut Pass);
 
     /// Writes a module to the specified path. Returns 0 on success.
@@ -1118,8 +1200,10 @@ extern "C" {
     pub(crate) fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint;
 
     // Operations on real types
+    pub(crate) fn LLVMHalfTypeInContext(C: &Context) -> &Type;
     pub(crate) fn LLVMFloatTypeInContext(C: &Context) -> &Type;
     pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type;
+    pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type;
 
     // Operations on function types
     pub(crate) fn LLVMFunctionType<'a>(
@@ -1141,6 +1225,7 @@ extern "C" {
     pub(crate) fn LLVMGetStructElementTypes<'a>(StructTy: &'a Type, Dest: *mut &'a Type);
     pub(crate) fn LLVMCountStructElementTypes(StructTy: &Type) -> c_uint;
     pub(crate) fn LLVMIsPackedStruct(StructTy: &Type) -> Bool;
+    pub(crate) fn LLVMStructGetTypeAtIndex(StructTy: &Type, i: c_uint) -> &Type;
 
     // Operations on array, pointer, and vector types (sequence types)
     pub(crate) fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type;
@@ -1149,13 +1234,13 @@ extern "C" {
 
     pub(crate) fn LLVMGetElementType(Ty: &Type) -> &Type;
     pub(crate) fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint;
-    pub(crate) fn LLVMRustGetValueType(V: &Value) -> &Type;
 
     // Operations on other types
     pub(crate) fn LLVMVoidTypeInContext(C: &Context) -> &Type;
     pub(crate) fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type;
 
     // Operations on all values
+    pub(crate) fn LLVMIsUndef(Val: &Value) -> Bool;
     pub(crate) fn LLVMTypeOf(Val: &Value) -> &Type;
     pub(crate) fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char;
     pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t);
@@ -1218,7 +1303,8 @@ extern "C" {
     pub(crate) fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value;
 
     // Constant expressions
-    pub(crate) fn LLVMConstInBoundsGEP<'a>(
+    pub(crate) fn LLVMConstInBoundsGEP2<'a>(
+        Ty: &'a Type,
         ConstantVal: &'a Value,
         ConstantIndices: *const &'a Value,
         NumIndices: c_uint,
@@ -1274,7 +1360,9 @@ extern "C" {
         Fn: &Value,
         index: c_uint,
         Name: *const c_char,
+        NameLen: size_t,
         Value: *const c_char,
+        ValueLen: size_t,
     );
     pub(crate) fn LLVMRustRemoveFunctionAttributes(Fn: &Value, index: c_uint, attr: Attribute);
 
@@ -1503,15 +1591,16 @@ extern "C" {
         Name: *const c_char,
     ) -> &'a Value;
     pub(crate) fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char)
-        -> &'a Value;
+    -> &'a Value;
     pub(crate) fn LLVMBuildFNeg<'a>(
         B: &Builder<'a>,
         V: &'a Value,
         Name: *const c_char,
     ) -> &'a Value;
     pub(crate) fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char)
-        -> &'a Value;
+    -> &'a Value;
     pub(crate) fn LLVMRustSetFastMath(Instr: &Value);
+    pub(crate) fn LLVMRustSetAlgebraicMath(Instr: &Value);
 
     // Memory
     pub(crate) fn LLVMBuildAlloca<'a>(
@@ -1533,26 +1622,22 @@ extern "C" {
 
     pub(crate) fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value;
 
-    pub(crate) fn LLVMBuildGEP<'a>(
+    pub(crate) fn LLVMBuildGEP2<'a>(
         B: &Builder<'a>,
+        Ty: &'a Type,
         Pointer: &'a Value,
         Indices: *const &'a Value,
         NumIndices: c_uint,
         Name: *const c_char,
     ) -> &'a Value;
-    pub(crate) fn LLVMBuildInBoundsGEP<'a>(
+    pub(crate) fn LLVMBuildInBoundsGEP2<'a>(
         B: &Builder<'a>,
+        Ty: &'a Type,
         Pointer: &'a Value,
         Indices: *const &'a Value,
         NumIndices: c_uint,
         Name: *const c_char,
     ) -> &'a Value;
-    pub(crate) fn LLVMBuildStructGEP<'a>(
-        B: &Builder<'a>,
-        Pointer: &'a Value,
-        Idx: c_uint,
-        Name: *const c_char,
-    ) -> &'a Value;
 
     // Casts
     pub(crate) fn LLVMBuildTrunc<'a>(
@@ -1658,7 +1743,7 @@ extern "C" {
 
     // Miscellaneous instructions
     pub(crate) fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char)
-        -> &'a Value;
+    -> &'a Value;
     pub(crate) fn LLVMRustGetInstrProfIncrementIntrinsic<'a>(M: &Module) -> &'a Value;
     pub(crate) fn LLVMRustBuildMemCpy<'a>(
         B: &Builder<'a>,
@@ -1816,6 +1901,7 @@ extern "C" {
     pub(crate) fn LLVMRustPrintModule<'a>(
         M: &'a Module,
         Output: *const c_char,
+        OutputLen: size_t,
         Demangle: extern "C" fn(*const c_char, size_t, *mut c_char, size_t) -> size_t,
     ) -> LLVMRustResult;
 
@@ -1843,6 +1929,7 @@ extern "C" {
         Data: *const u8,
         len: usize,
         Identifier: *const c_char,
+        IdentiferLen: size_t,
     ) -> Option<&Module>;
     pub(crate) fn LLVMRustGetBitcodeSliceFromObjectData(
         Data: *const u8,
diff --git a/crates/rustc_codegen_nvvm/src/lto.rs b/crates/rustc_codegen_nvvm/src/lto.rs
index 74c51d6f..9e124a65 100644
--- a/crates/rustc_codegen_nvvm/src/lto.rs
+++ b/crates/rustc_codegen_nvvm/src/lto.rs
@@ -1,21 +1,21 @@
-use crate::{llvm, LlvmMod};
+use std::ffi::CString;
+use std::sync::Arc;
+
 use rustc_codegen_ssa::{
+    ModuleCodegen,
     back::{
         lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared},
         write::CodegenContext,
     },
     traits::{ModuleBufferMethods, ThinBufferMethods},
-    ModuleCodegen, ModuleKind,
 };
-use rustc_errors::{FatalError, Handler};
+use rustc_errors::{DiagCtxtHandle, FatalError};
 use rustc_middle::dep_graph::WorkProduct;
-use std::{
-    ffi::{CStr, CString},
-    sync::Arc,
-};
 use tracing::{debug, trace};
 
 use crate::NvvmCodegenBackend;
+use crate::common::AsCCharPtr;
+use crate::{LlvmMod, llvm};
 
 pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer);
 
@@ -73,6 +73,10 @@ impl ThinBufferMethods for ThinBuffer {
             std::slice::from_raw_parts(ptr, len)
         }
     }
+
+    fn thin_link_data(&self) -> &[u8] {
+        todo!()
+    }
 }
 
 impl Drop for ThinBuffer {
@@ -148,35 +152,40 @@ pub(crate) fn run_thin(
 
 pub(crate) unsafe fn optimize_thin(
     cgcx: &CodegenContext<NvvmCodegenBackend>,
-    thin_module: &mut ThinModule<NvvmCodegenBackend>,
+    thin_module: ThinModule<NvvmCodegenBackend>,
 ) -> Result<ModuleCodegen<LlvmMod>, FatalError> {
     // essentially does nothing
-    let diag_handler = cgcx.create_diag_handler();
+    let dcx = cgcx.create_dcx();
+    let dcx = dcx.handle();
+
     let module_name = &thin_module.shared.module_names[thin_module.idx];
 
-    let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
-    let llmod = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _;
+    let llcx = unsafe { llvm::LLVMRustContextCreate(cgcx.fewer_names) };
+    let llmod =
+        parse_module(llcx, module_name.to_str().unwrap(), thin_module.data(), dcx)? as *const _;
 
-    let module = ModuleCodegen {
-        module_llvm: LlvmMod { llmod, llcx },
-        name: thin_module.name().to_string(),
-        kind: ModuleKind::Regular,
-    };
+    let module =
+        ModuleCodegen::new_regular(thin_module.name().to_string(), LlvmMod { llcx, llmod });
     Ok(module)
 }
 
 pub(crate) fn parse_module<'a>(
     cx: &'a llvm::Context,
-    name: &CStr,
+    name: &str,
     data: &[u8],
-    diag_handler: &Handler,
+    dcx: DiagCtxtHandle<'_>,
 ) -> Result<&'a llvm::Module, FatalError> {
     unsafe {
-        llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else(
-            || {
-                let msg = "failed to parse bitcode for LTO module";
-                crate::back::llvm_err(diag_handler, msg)
-            },
+        llvm::LLVMRustParseBitcodeForLTO(
+            cx,
+            data.as_ptr(),
+            data.len(),
+            name.as_c_char_ptr(),
+            name.len(),
         )
+        .ok_or_else(|| {
+            let msg = "failed to parse bitcode for LTO module";
+            crate::back::llvm_err(dcx, msg)
+        })
     }
 }
diff --git a/crates/rustc_codegen_nvvm/src/mono_item.rs b/crates/rustc_codegen_nvvm/src/mono_item.rs
index a84c7579..0cc70708 100644
--- a/crates/rustc_codegen_nvvm/src/mono_item.rs
+++ b/crates/rustc_codegen_nvvm/src/mono_item.rs
@@ -7,11 +7,10 @@ use crate::llvm;
 use crate::ty::LayoutLlvmExt;
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
-pub use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::mir::mono::{Linkage, Visibility};
-use rustc_middle::ty::layout::FnAbiOf;
-use rustc_middle::ty::layout::LayoutOf;
-use rustc_middle::ty::{self, Instance, TypeFoldable};
+use rustc_middle::ty::TypeVisitableExt;
+use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf};
+use rustc_middle::ty::{self, Instance};
 use tracing::trace;
 
 pub(crate) fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility {
@@ -22,7 +21,7 @@ pub(crate) fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility {
     }
 }
 
-impl<'ll, 'tcx> PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn predefine_static(
         &self,
         def_id: DefId,
@@ -32,16 +31,16 @@ impl<'ll, 'tcx> PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     ) {
         trace!("Predefining static with name `{}`", symbol_name);
         let instance = Instance::mono(self.tcx, def_id);
-        let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
+        let ty = instance.ty(self.tcx, self.typing_env());
         let llty = self.layout_of(ty).llvm_type(self);
         let addrspace = self.static_addrspace(instance);
 
         let g = self
             .define_global(symbol_name, llty, addrspace)
             .unwrap_or_else(|| {
-                self.sess().span_fatal(
+                self.sess().dcx().span_fatal(
                     self.tcx.def_span(def_id),
-                    &format!("symbol `{}` is already defined", symbol_name),
+                    format!("symbol `{}` is already defined", symbol_name),
                 )
             });
 
@@ -66,7 +65,7 @@ impl<'ll, 'tcx> PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             linkage,
             self.tcx.codegen_fn_attrs(instance.def_id())
         );
-        assert!(!instance.substs.needs_infer());
+        assert!(!instance.args.has_infer());
 
         let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
 
@@ -78,10 +77,7 @@ impl<'ll, 'tcx> PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         // compiler-rt, then we want to implicitly compile everything with hidden
         // visibility as we're going to link this object all over the place but
         // don't want the symbols to get exported.
-        if linkage != Linkage::Internal
-            && linkage != Linkage::Private
-            && self.tcx.is_compiler_builtins(LOCAL_CRATE)
-        {
+        if linkage != Linkage::Internal && self.tcx.is_compiler_builtins(LOCAL_CRATE) {
             unsafe {
                 llvm::LLVMRustSetVisibility(lldecl, llvm::Visibility::Hidden);
             }
@@ -94,7 +90,7 @@ impl<'ll, 'tcx> PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         attributes::from_fn_attrs(self, lldecl, instance);
 
         let def_id = instance.def_id();
-        let attrs = self.tcx.get_attrs(def_id);
+        let attrs = self.tcx.get_attrs_unchecked(def_id); // TODO: Replace with get_attrs
         let nvvm_attrs = NvvmAttributes::parse(self, attrs);
 
         unsafe {
diff --git a/crates/rustc_codegen_nvvm/src/nvvm.rs b/crates/rustc_codegen_nvvm/src/nvvm.rs
index 00f2c84d..e6ca2c47 100644
--- a/crates/rustc_codegen_nvvm/src/nvvm.rs
+++ b/crates/rustc_codegen_nvvm/src/nvvm.rs
@@ -2,14 +2,14 @@
 
 use crate::back::demangle_callback;
 use crate::builder::unnamed;
+use crate::common::AsCCharPtr;
 use crate::context::CodegenArgs;
 use crate::llvm::*;
 use crate::lto::ThinBuffer;
 use find_cuda_helper::find_cuda_root;
 use nvvm::*;
 use rustc_codegen_ssa::traits::ThinBufferMethods;
-use rustc_fs_util::path_to_c_string;
-use rustc_session::{config::DebugInfo, Session};
+use rustc_session::{Session, config::DebugInfo};
 use std::ffi::OsStr;
 use std::fmt::Display;
 use std::marker::PhantomData;
@@ -64,7 +64,8 @@ pub fn codegen_bitcode_modules(
     let (major, minor) = nvvm::ir_version();
 
     if minor < 6 || major < 1 {
-        sess.fatal("rustc_codegen_nvvm requires at least libnvvm 1.6 (CUDA 11.2)");
+        sess.dcx()
+            .fatal("rustc_codegen_nvvm requires at least libnvvm 1.6 (CUDA 11.2)");
     }
 
     // first, create the nvvm program we will add modules to.
@@ -94,8 +95,9 @@ pub fn codegen_bitcode_modules(
         LLVMAddNamedMetadataOperand(module, "nvvmir.version\0".as_ptr().cast(), node);
 
         if let Some(path) = &args.final_module_path {
-            let out_c = path_to_c_string(path);
-            let result = LLVMRustPrintModule(module, out_c.as_ptr(), demangle_callback);
+            let out = path.to_str().unwrap();
+            let result =
+                LLVMRustPrintModule(module, out.as_c_char_ptr(), out.len(), demangle_callback);
             result
                 .into_result()
                 .expect("Failed to write final llvm module output");
@@ -112,7 +114,8 @@ pub fn codegen_bitcode_modules(
         // i would put a more helpful error here, but to actually use the codegen
         // it needs to find libnvvm before this, and libdevice is in the nvvm directory
         // so if it can find libnvvm there is almost no way it can't find libdevice.
-        sess.fatal("Could not find the libdevice library (libdevice.10.bc) in the CUDA directory")
+        sess.dcx()
+            .fatal("Could not find the libdevice library (libdevice.10.bc) in the CUDA directory")
     };
 
     prog.add_lazy_module(&libdevice, "libdevice".to_string())?;
@@ -165,10 +168,12 @@ pub fn find_libdevice() -> Option<Vec<u8>> {
 }
 
 unsafe fn cleanup_dicompileunit(module: &Module) {
-    let mut cu1 = ptr::null_mut();
-    let mut cu2 = ptr::null_mut();
-    LLVMRustThinLTOGetDICompileUnit(module, &mut cu1, &mut cu2);
-    LLVMRustThinLTOPatchDICompileUnit(module, cu1);
+    unsafe {
+        let mut cu1 = ptr::null_mut();
+        let mut cu2 = ptr::null_mut();
+        LLVMRustThinLTOGetDICompileUnit(module, &mut cu1, &mut cu2);
+        LLVMRustThinLTOPatchDICompileUnit(module, cu1);
+    }
 }
 
 // Merging and DCE (dead code elimination) logic. Inspired a lot by rust-ptx-linker.
@@ -191,6 +196,7 @@ fn merge_llvm_modules(modules: Vec<Vec<u8>>, llcx: &Context) -> &Module {
                 merged_module.as_ptr(),
                 merged_module.len(),
                 unnamed(),
+                0,
             )
             .expect("Failed to parse module bitcode");
             LLVMLinkModules2(module, tmp);
@@ -258,85 +264,89 @@ impl<'a, 'll> Iterator for GlobalIter<'a, 'll> {
 }
 
 unsafe fn internalize_pass(module: &Module, cx: &Context) {
-    // collect the values of all the declared kernels
-    let num_operands =
-        LLVMGetNamedMetadataNumOperands(module, "nvvm.annotations\0".as_ptr().cast()) as usize;
-    let mut operands = Vec::with_capacity(num_operands);
-    LLVMGetNamedMetadataOperands(
-        module,
-        "nvvm.annotations\0".as_ptr().cast(),
-        operands.as_mut_ptr(),
-    );
-    operands.set_len(num_operands);
-    let mut kernels = Vec::with_capacity(num_operands);
-    let kernel_str = LLVMMDStringInContext(cx, "kernel".as_ptr().cast(), 6);
-
-    for mdnode in operands {
-        let num_operands = LLVMGetMDNodeNumOperands(mdnode) as usize;
+    unsafe {
+        // collect the values of all the declared kernels
+        let num_operands =
+            LLVMGetNamedMetadataNumOperands(module, "nvvm.annotations\0".as_ptr().cast()) as usize;
         let mut operands = Vec::with_capacity(num_operands);
-        LLVMGetMDNodeOperands(mdnode, operands.as_mut_ptr());
+        LLVMGetNamedMetadataOperands(
+            module,
+            "nvvm.annotations\0".as_ptr().cast(),
+            operands.as_mut_ptr(),
+        );
         operands.set_len(num_operands);
-
-        if operands.get(1) == Some(&kernel_str) {
-            kernels.push(operands[0]);
+        let mut kernels = Vec::with_capacity(num_operands);
+        let kernel_str = LLVMMDStringInContext(cx, "kernel".as_ptr().cast(), 6);
+
+        for mdnode in operands {
+            let num_operands = LLVMGetMDNodeNumOperands(mdnode) as usize;
+            let mut operands = Vec::with_capacity(num_operands);
+            LLVMGetMDNodeOperands(mdnode, operands.as_mut_ptr());
+            operands.set_len(num_operands);
+
+            if operands.get(1) == Some(&kernel_str) {
+                kernels.push(operands[0]);
+            }
         }
-    }
 
-    // see what functions are marked as externally visible by the user.
-    let num_operands =
-        LLVMGetNamedMetadataNumOperands(module, "cg_nvvm_used\0".as_ptr().cast()) as usize;
-    let mut operands = Vec::with_capacity(num_operands);
-    LLVMGetNamedMetadataOperands(
-        module,
-        "cg_nvvm_used\0".as_ptr().cast(),
-        operands.as_mut_ptr(),
-    );
-    operands.set_len(num_operands);
-    let mut used_funcs = Vec::with_capacity(num_operands);
-
-    for mdnode in operands {
-        let num_operands = LLVMGetMDNodeNumOperands(mdnode) as usize;
+        // see what functions are marked as externally visible by the user.
+        let num_operands =
+            LLVMGetNamedMetadataNumOperands(module, "cg_nvvm_used\0".as_ptr().cast()) as usize;
         let mut operands = Vec::with_capacity(num_operands);
-        LLVMGetMDNodeOperands(mdnode, operands.as_mut_ptr());
+        LLVMGetNamedMetadataOperands(
+            module,
+            "cg_nvvm_used\0".as_ptr().cast(),
+            operands.as_mut_ptr(),
+        );
         operands.set_len(num_operands);
+        let mut used_funcs = Vec::with_capacity(num_operands);
 
-        used_funcs.push(operands[0]);
-    }
-
-    let iter = FunctionIter::new(&module);
-    for func in iter {
-        let is_kernel = kernels.contains(&func);
-        let is_decl = LLVMIsDeclaration(func) == True;
-        let is_used = used_funcs.contains(&func);
+        for mdnode in operands {
+            let num_operands = LLVMGetMDNodeNumOperands(mdnode) as usize;
+            let mut operands = Vec::with_capacity(num_operands);
+            LLVMGetMDNodeOperands(mdnode, operands.as_mut_ptr());
+            operands.set_len(num_operands);
 
-        if !is_decl && !is_kernel {
-            LLVMRustSetLinkage(func, Linkage::InternalLinkage);
-            LLVMRustSetVisibility(func, Visibility::Default);
+            used_funcs.push(operands[0]);
         }
 
-        // explicitly set it to external just in case the codegen set them to internal for some reason
-        if is_used {
-            LLVMRustSetLinkage(func, Linkage::ExternalLinkage);
-            LLVMRustSetVisibility(func, Visibility::Default);
+        let iter = FunctionIter::new(&module);
+        for func in iter {
+            let is_kernel = kernels.contains(&func);
+            let is_decl = LLVMIsDeclaration(func) == True;
+            let is_used = used_funcs.contains(&func);
+
+            if !is_decl && !is_kernel {
+                LLVMRustSetLinkage(func, Linkage::InternalLinkage);
+                LLVMRustSetVisibility(func, Visibility::Default);
+            }
+
+            // explicitly set it to external just in case the codegen set them to internal for some reason
+            if is_used {
+                LLVMRustSetLinkage(func, Linkage::ExternalLinkage);
+                LLVMRustSetVisibility(func, Visibility::Default);
+            }
         }
-    }
 
-    let iter = GlobalIter::new(&module);
-    for func in iter {
-        let is_decl = LLVMIsDeclaration(func) == True;
+        let iter = GlobalIter::new(&module);
+        for func in iter {
+            let is_decl = LLVMIsDeclaration(func) == True;
 
-        if !is_decl {
-            LLVMRustSetLinkage(func, Linkage::InternalLinkage);
-            LLVMRustSetVisibility(func, Visibility::Default);
+            if !is_decl {
+                LLVMRustSetLinkage(func, Linkage::InternalLinkage);
+                LLVMRustSetVisibility(func, Visibility::Default);
+            }
         }
     }
 }
 
 unsafe fn dce_pass(module: &Module) {
-    let pass_manager = LLVMCreatePassManager();
+    unsafe {
+        let pass_manager = LLVMCreatePassManager();
 
-    LLVMAddGlobalDCEPass(pass_manager);
+        LLVMAddGlobalDCEPass(pass_manager);
 
-    LLVMRunPassManager(pass_manager, module);
-    LLVMDisposePassManager(pass_manager);
+        LLVMRunPassManager(pass_manager, module);
+        LLVMDisposePassManager(pass_manager);
+    }
 }
diff --git a/crates/rustc_codegen_nvvm/src/override_fns.rs b/crates/rustc_codegen_nvvm/src/override_fns.rs
index 61a8ef3c..7c334297 100644
--- a/crates/rustc_codegen_nvvm/src/override_fns.rs
+++ b/crates/rustc_codegen_nvvm/src/override_fns.rs
@@ -4,7 +4,7 @@
 
 use crate::{builder::Builder, context::CodegenCx, llvm};
 use rustc_codegen_ssa::mono_item::MonoItemExt;
-use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods};
+use rustc_codegen_ssa::traits::BuilderMethods;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::{mir::mono::MonoItem, ty::Instance};
 
@@ -27,7 +27,8 @@ fn should_override<'ll, 'tcx>(func: Instance<'tcx>, cx: &CodegenCx<'ll, 'tcx>) -
     if !is_libm {
         return false;
     }
-    let name = cx.tcx.item_name(func.def_id()).as_str();
+    let sym = cx.tcx.item_name(func.def_id());
+    let name = sym.as_str();
     let intrinsics = cx.intrinsics_map.borrow();
     let is_known_intrinsic = intrinsics.contains_key(format!("__nv_{}", name).as_str());
 
@@ -48,15 +49,23 @@ fn is_unsupported_libdevice_fn(name: &str) -> bool {
 }
 
 fn override_libm_function<'ll, 'tcx>(func: Instance<'tcx>, cx: &CodegenCx<'ll, 'tcx>) {
-    let name = cx.tcx.item_name(func.def_id()).as_str();
-    let nv_name = format!("__nv_{}", name);
-    let intrinsic = cx.get_intrinsic(&nv_name);
+    let name = cx.tcx.item_name(func.def_id());
+    let nv_name = format!("__nv_{}", name.as_str());
+    let (intrinsic_llfn_ty, intrinsic_llfn) = cx.get_intrinsic(nv_name.as_str());
 
     let llfn = cx.get_fn(func);
     let start = Builder::append_block(cx, llfn, "start");
     let mut bx = Builder::build(cx, start);
 
     let params = llvm::get_params(llfn);
-    let llcall = bx.call(cx.type_i1(), intrinsic, &params, None);
+    let llcall = bx.call(
+        intrinsic_llfn_ty,
+        None,
+        None,
+        intrinsic_llfn,
+        &params,
+        None,
+        None,
+    );
     bx.ret(llcall);
 }
diff --git a/crates/rustc_codegen_nvvm/src/target.rs b/crates/rustc_codegen_nvvm/src/target.rs
index dc01761d..9cd09997 100644
--- a/crates/rustc_codegen_nvvm/src/target.rs
+++ b/crates/rustc_codegen_nvvm/src/target.rs
@@ -1,5 +1,7 @@
 use crate::llvm::{self, Type};
-use rustc_target::spec::{LinkerFlavor, MergeFunctions, PanicStrategy, Target, TargetOptions};
+use rustc_target::spec::{
+    LinkerFlavor, MergeFunctions, PanicStrategy, Target, TargetMetadata, TargetOptions,
+};
 
 pub const DATA_LAYOUT: &str = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64";
 pub const TARGET_TRIPLE: &str = "nvptx64-nvidia-cuda";
@@ -7,48 +9,46 @@ pub const POINTER_WIDTH: u32 = 64;
 
 /// The pointer width of the current target
 pub(crate) unsafe fn usize_ty(llcx: &'_ llvm::Context) -> &'_ Type {
-    llvm::LLVMInt64TypeInContext(llcx)
+    unsafe { llvm::LLVMInt64TypeInContext(llcx) }
 }
 
 pub fn target() -> Target {
-    Target {
-        arch: "nvptx".to_string(),
-        data_layout: DATA_LAYOUT.to_string(),
-        llvm_target: "nvptx64-nvidia-cuda".to_string(),
-        pointer_width: 64,
-
-        options: TargetOptions {
-            os: "cuda".to_string(),
-            vendor: "nvidia".to_string(),
-            linker_flavor: LinkerFlavor::PtxLinker,
-            // nvvm does all the linking for us, but technically its not a linker
-            linker: None,
-
-            cpu: "sm_30".to_string(),
-
-            max_atomic_width: Some(64),
-
-            // Unwinding on CUDA is neither feasible nor useful.
-            panic_strategy: PanicStrategy::Abort,
-
-            // Needed to use `dylib` and `bin` crate types and the linker.
-            dynamic_linking: true,
-            executables: true,
-
-            only_cdylib: true,
-
-            // nvvm does all the work of turning the bitcode into ptx
-            obj_is_bitcode: true,
-
-            dll_prefix: "".to_string(),
-            dll_suffix: ".ptx".to_string(),
-            exe_suffix: ".ptx".to_string(),
-
-            // Disable MergeFunctions LLVM optimisation pass because it can
-            // produce kernel functions that call other kernel functions.
-            // This behavior is not supported by PTX ISA.
-            merge_functions: MergeFunctions::Disabled,
+    let mut options = TargetOptions::default();
+
+    options.os = "cuda".into();
+    options.vendor = "nvidia".into();
+    options.linker_flavor = LinkerFlavor::Ptx;
+    // nvvm does all the linking for us, but technically its not a linker
+    options.linker = None;
+    options.cpu = "sm_30".into();
+    options.max_atomic_width = Some(64);
+    // Unwinding on CUDA is neither feasible nor useful.
+    options.panic_strategy = PanicStrategy::Abort;
+    // Needed to use `dylib` and `bin` crate types and the linker.
+    options.dynamic_linking = true;
+    options.executables = true;
+    options.only_cdylib = true;
+
+    // nvvm does all the work of turning the bitcode into ptx
+    options.obj_is_bitcode = true;
+
+    options.dll_prefix = "".into();
+    options.dll_suffix = ".ptx".into();
+    options.exe_suffix = ".ptx".into();
+
+    // Disable MergeFunctions LLVM optimisation pass because it can
+    // produce kernel functions that call other kernel functions.
+    // This behavior is not supported by PTX ISA.
+    options.merge_functions = MergeFunctions::Disabled;
 
+    Target {
+        arch: "nvptx".into(),
+        data_layout: DATA_LAYOUT.into(),
+        llvm_target: "nvptx64-nvidia-cuda".into(),
+        pointer_width: POINTER_WIDTH,
+        options,
+        metadata: TargetMetadata {
+            description: Some("NVIDIA CUDA".into()),
             ..Default::default()
         },
     }
diff --git a/crates/rustc_codegen_nvvm/src/ty.rs b/crates/rustc_codegen_nvvm/src/ty.rs
index 55102dba..9052ec3e 100644
--- a/crates/rustc_codegen_nvvm/src/ty.rs
+++ b/crates/rustc_codegen_nvvm/src/ty.rs
@@ -1,26 +1,20 @@
 use crate::abi::{FnAbiLlvmExt, LlvmType};
 use crate::context::CodegenCx;
 use crate::llvm::{self, Bool, False, True, Type, Value};
-use crate::rustc_target::abi::TyAbiInterface;
 use libc::c_uint;
+use rustc_abi::Primitive::{Float, Int, Pointer};
+use rustc_abi::{
+    AddressSpace, Align, BackendRepr, FieldsShape, Integer, PointeeInfo, Reg, Scalar, Size,
+    TyAbiInterface, Variants,
+};
 use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::traits::*;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_middle::bug;
-use rustc_middle::ty;
-use rustc_middle::ty::layout::TyAndLayout;
-use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
+use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
 use rustc_middle::ty::print::with_no_trimmed_paths;
-use rustc_middle::ty::Ty;
-use rustc_middle::ty::TypeFoldable;
-use rustc_target::abi::call::{CastTarget, FnAbi, Reg};
-use rustc_target::abi::Abi;
-use rustc_target::abi::FieldsShape;
-use rustc_target::abi::PointeeInfo;
-use rustc_target::abi::Primitive::*;
-use rustc_target::abi::Scalar;
-use rustc_target::abi::Variants;
-use rustc_target::abi::{AddressSpace, Align, Integer, Size};
+use rustc_middle::ty::{self, CoroutineArgsExt, Ty, TypeVisitableExt};
+use rustc_target::callconv::{CastTarget, FnAbi};
 use std::ffi::CString;
 use std::fmt::{Debug, Write};
 use std::hash::Hash;
@@ -71,6 +65,17 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         unsafe { llvm::LLVMStructCreateNamed(self.llcx, name.as_ptr()) }
     }
 
+    pub(crate) fn type_struct(&self, els: &[&'ll Type], packed: bool) -> &'ll Type {
+        unsafe {
+            llvm::LLVMStructTypeInContext(
+                self.llcx,
+                els.as_ptr(),
+                els.len() as c_uint,
+                packed as Bool,
+            )
+        }
+    }
+
     pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) {
         unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) }
     }
@@ -83,6 +88,18 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) }
     }
 
+    pub(crate) fn type_i1(&self) -> &'ll Type {
+        unsafe { llvm::LLVMInt1TypeInContext(self.llcx) }
+    }
+
+    pub(crate) fn type_i8p(&self) -> &'ll Type {
+        self.type_i8p_ext(AddressSpace::DATA)
+    }
+
+    pub(crate) fn type_i8p_ext(&self, address_space: AddressSpace) -> &'ll Type {
+        self.type_ptr_to_ext(self.type_i8(), address_space)
+    }
+
     ///x Creates an integer type with the given number of bits, e.g., i24
     pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type {
         unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) }
@@ -92,6 +109,20 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         unsafe { llvm::LLVMVectorType(ty, len as c_uint) }
     }
 
+    pub(crate) fn type_ptr_to(&self, ty: &'ll Type) -> &'ll Type {
+        assert_ne!(
+            self.type_kind(ty),
+            TypeKind::Function,
+            "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense"
+        );
+
+        unsafe { llvm::LLVMPointerType(ty, AddressSpace::DATA.0) }
+    }
+
+    pub(crate) fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type {
+        unsafe { llvm::LLVMPointerType(ty, address_space.0) }
+    }
+
     pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> {
         unsafe {
             let n_args = llvm::LLVMCountParamTypes(ty) as usize;
@@ -126,11 +157,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     }
 }
 
-impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
-    fn type_i1(&self) -> &'ll Type {
-        unsafe { llvm::LLVMInt1TypeInContext(self.llcx) }
-    }
-
+impl<'ll, 'tcx> BaseTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn type_i8(&self) -> &'ll Type {
         unsafe { llvm::LLVMInt8TypeInContext(self.llcx) }
     }
@@ -155,6 +182,10 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         self.isize_ty
     }
 
+    fn type_f16(&self) -> &'ll Type {
+        unsafe { llvm::LLVMHalfTypeInContext(self.llcx) }
+    }
+
     fn type_f32(&self) -> &'ll Type {
         unsafe { llvm::LLVMFloatTypeInContext(self.llcx) }
     }
@@ -163,37 +194,28 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         unsafe { llvm::LLVMDoubleTypeInContext(self.llcx) }
     }
 
-    fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type {
-        unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) }
+    fn type_f128(&self) -> &'ll Type {
+        unsafe { llvm::LLVMFP128TypeInContext(self.llcx) }
     }
 
-    fn type_struct(&self, els: &[&'ll Type], packed: bool) -> &'ll Type {
-        unsafe {
-            llvm::LLVMStructTypeInContext(
-                self.llcx,
-                els.as_ptr(),
-                els.len() as c_uint,
-                packed as Bool,
-            )
-        }
+    fn type_array(&self, ty: Self::Type, len: u64) -> Self::Type {
+        unsafe { llvm::LLVMRustArrayType(ty, len) }
+    }
+
+    fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type {
+        unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) }
     }
 
     fn type_kind(&self, ty: &'ll Type) -> TypeKind {
         unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() }
     }
 
-    fn type_ptr_to(&self, ty: &'ll Type) -> &'ll Type {
-        assert_ne!(
-            self.type_kind(ty),
-            TypeKind::Function,
-            "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense"
-        );
-
-        unsafe { llvm::LLVMPointerType(ty, AddressSpace::DATA.0) }
+    fn type_ptr(&self) -> Self::Type {
+        self.type_ptr_ext(AddressSpace::DATA)
     }
 
-    fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type {
-        unsafe { llvm::LLVMPointerType(ty, address_space.0) }
+    fn type_ptr_ext(&self, address_space: AddressSpace) -> Self::Type {
+        self.type_ptr_to_ext(self.type_i8(), address_space)
     }
 
     fn element_type(&self, ty: &'ll Type) -> &'ll Type {
@@ -224,7 +246,7 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     }
 }
 
-impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type {
         layout.llvm_type(self)
     }
@@ -237,9 +259,6 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool {
         layout.is_llvm_scalar_pair()
     }
-    fn backend_field_index(&self, layout: TyAndLayout<'tcx>, index: usize) -> u64 {
-        layout.llvm_field_index(index)
-    }
     fn scalar_pair_element_backend_type(
         &self,
         layout: TyAndLayout<'tcx>,
@@ -268,35 +287,31 @@ pub(crate) trait LayoutLlvmExt<'tcx> {
     fn is_llvm_scalar_pair(&self) -> bool;
     fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type;
     fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type;
-    fn scalar_llvm_type_at<'a>(
-        &self,
-        cx: &CodegenCx<'a, 'tcx>,
-        scalar: &Scalar,
-        offset: Size,
-    ) -> &'a Type;
+    fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type;
     fn scalar_pair_element_llvm_type<'a>(
         &self,
         cx: &CodegenCx<'a, 'tcx>,
         index: usize,
         immediate: bool,
     ) -> &'a Type;
-    fn llvm_field_index(&self, index: usize) -> u64;
     fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option<PointeeInfo>;
 }
 
 impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
     fn is_llvm_immediate(&self) -> bool {
-        match self.abi {
-            Abi::Scalar(_) | Abi::Vector { .. } => true,
-            Abi::ScalarPair(..) => false,
-            Abi::Uninhabited | Abi::Aggregate { .. } => self.is_zst(),
+        match self.backend_repr {
+            BackendRepr::Scalar(_) | BackendRepr::Vector { .. } => true,
+            BackendRepr::ScalarPair(..) => false,
+            BackendRepr::Memory { .. } => self.is_zst(),
         }
     }
 
     fn is_llvm_scalar_pair(&self) -> bool {
-        match self.abi {
-            Abi::ScalarPair(..) => true,
-            Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false,
+        match self.backend_repr {
+            BackendRepr::ScalarPair(..) => true,
+            BackendRepr::Scalar(_) | BackendRepr::Vector { .. } | BackendRepr::Memory { .. } => {
+                false
+            }
         }
     }
 
@@ -312,7 +327,10 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
     /// of that field's type - this is useful for taking the address of
     /// that field and ensuring the struct has the right alignment.
     fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type {
-        if let Abi::Scalar(ref scalar) = self.abi {
+        // This must produce the same result for `repr(transparent)` wrappers as for the inner type!
+        // In other words, this should generally not look at the type at all, but only at the
+        // layout.
+        if let BackendRepr::Scalar(scalar) = self.backend_repr {
             // Use a different cache for scalars because pointers to DSTs
             // can be either fat or thin (data pointers of fat pointers).
             if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) {
@@ -320,19 +338,18 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
             }
 
             let llty = match *self.ty.kind() {
-                ty::Ref(_, ty, _) | ty::RawPtr(ty::TypeAndMut { ty, .. }) => {
+                ty::Ref(_, ty, _) | ty::RawPtr(ty, _) => {
                     cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx))
                 }
                 ty::Adt(def, _) if def.is_box() => {
-                    cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx))
+                    cx.type_ptr_to(cx.layout_of(self.ty.expect_boxed_ty()).llvm_type(cx))
                 }
-                ty::FnPtr(sig) => {
-                    cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig, ty::List::empty()))
+                ty::FnPtr(sig, hdr) => {
+                    cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig.with(hdr), ty::List::empty()))
                 }
-                _ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO),
+                _ => self.scalar_llvm_type_at(cx, scalar),
             };
             cx.scalar_lltypes.borrow_mut().insert(self.ty, llty);
-
             return llty;
         }
 
@@ -344,6 +361,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
         if let Some(&llty) = cx.lltypes.borrow().get(&(self.ty, variant_index)) {
             return llty;
         }
+
         assert!(
             !self.ty.has_escaping_bound_vars(),
             "{:?} has escaping bound vars",
@@ -353,6 +371,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
         // Make sure lifetimes are erased, to avoid generating distinct LLVM
         // types for Rust types that only differ in the choice of lifetimes.
         let normal_ty = cx.tcx.erase_regions(self.ty);
+
         let mut defer = None;
         let llty = if self.ty != normal_ty {
             let mut layout = cx.layout_of(normal_ty);
@@ -363,6 +382,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
         } else {
             uncached_llvm_type(cx, *self, &mut defer)
         };
+
         cx.lltypes
             .borrow_mut()
             .insert((self.ty, variant_index), llty);
@@ -376,35 +396,44 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
     }
 
     fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type {
-        if let Abi::Scalar(ref scalar) = self.abi {
-            if scalar.is_bool() {
-                return cx.type_i1();
+        match self.backend_repr {
+            BackendRepr::Scalar(ref scalar) => {
+                if scalar.is_bool() {
+                    return cx.type_i1();
+                }
             }
-        }
+            BackendRepr::ScalarPair(..) => {
+                // An immediate pair always contains just the two elements, without any padding
+                // filler, as it should never be stored to memory.
+                return cx.type_struct(
+                    &[
+                        self.scalar_pair_element_llvm_type(cx, 0, true),
+                        self.scalar_pair_element_llvm_type(cx, 1, true),
+                    ],
+                    false,
+                );
+            }
+            _ => {}
+        };
         self.llvm_type(cx)
     }
 
-    fn scalar_llvm_type_at<'a>(
-        &self,
-        cx: &CodegenCx<'a, 'tcx>,
-        scalar: &Scalar,
-        offset: Size,
-    ) -> &'a Type {
-        match scalar.value {
+    fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type {
+        match scalar.primitive() {
             Int(i, _) => cx.type_from_integer(i),
-            F32 => cx.type_f32(),
-            F64 => cx.type_f64(),
-            Pointer => {
+            Float(f) => cx.type_from_float(f),
+            Pointer(address_space) => {
                 // If we know the alignment, pick something better than i8.
-                let (pointee, address_space) =
-                    if let Some(pointee) = self.pointee_info_at(cx, offset) {
-                        (
-                            cx.type_pointee_for_align(pointee.align),
-                            pointee.address_space,
-                        )
-                    } else {
-                        (cx.type_i8(), AddressSpace::DATA)
-                    };
+                let (pointee, address_space) = if let Some(PointeeInfo {
+                    safe: Some(_),
+                    align,
+                    ..
+                }) = self.pointee_info_at(cx, Size::ZERO)
+                {
+                    (cx.type_pointee_for_align(align), address_space)
+                } else {
+                    (cx.type_i8(), AddressSpace::DATA)
+                };
                 cx.type_ptr_to_ext(pointee, address_space)
             }
         }
@@ -416,14 +445,26 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
         index: usize,
         immediate: bool,
     ) -> &'a Type {
+        // This must produce the same result for `repr(transparent)` wrappers as for the inner type!
+        // In other words, this should generally not look at the type at all, but only at the
+        // layout.
+
         // HACK(eddyb) special-case fat pointers until LLVM removes
         // pointee types, to avoid bitcasting every `OperandRef::deref`.
         match self.ty.kind() {
-            ty::Ref(..) | ty::RawPtr(_) => {
+            ty::Ref(..) | ty::RawPtr(..) => {
                 return self.field(cx, index).llvm_type(cx);
             }
             ty::Adt(def, _) if def.is_box() => {
-                let ptr_ty = cx.tcx.mk_mut_ptr(self.ty.boxed_ty());
+                let ptr_ty = Ty::new_mut_ptr(cx.tcx, self.ty.boxed_ty().unwrap());
+                return cx
+                    .layout_of(ptr_ty)
+                    .scalar_pair_element_llvm_type(cx, index, immediate);
+            }
+            // `dyn* Trait` has the same ABI as `*mut dyn Trait`
+            ty::Dynamic(bounds, region, ty::DynStar) => {
+                let ptr_ty =
+                    Ty::new_mut_ptr(cx.tcx, Ty::new_dynamic(cx.tcx, bounds, *region, ty::Dyn));
                 return cx
                     .layout_of(ptr_ty)
                     .scalar_pair_element_llvm_type(cx, index, immediate);
@@ -431,12 +472,11 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
             _ => {}
         }
 
-        let (a, b) = match self.abi {
-            Abi::ScalarPair(ref a, ref b) => (a, b),
-            _ => bug!(
+        let BackendRepr::ScalarPair(a, b) = self.backend_repr else {
+            bug!(
                 "TyAndLayout::scalar_pair_element_llty({:?}): not applicable",
                 self
-            ),
+            );
         };
         let scalar = [a, b][index];
 
@@ -450,30 +490,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
             return cx.type_i1();
         }
 
-        let offset = if index == 0 {
-            Size::ZERO
-        } else {
-            a.value.size(cx).align_to(b.value.align(cx).abi)
-        };
-        self.scalar_llvm_type_at(cx, scalar, offset)
-    }
-
-    fn llvm_field_index(&self, index: usize) -> u64 {
-        match self.abi {
-            Abi::Scalar(_) | Abi::ScalarPair(..) => {
-                bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self)
-            }
-            _ => {}
-        }
-        match self.fields {
-            FieldsShape::Primitive | FieldsShape::Union(_) => {
-                bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self)
-            }
-
-            FieldsShape::Array { .. } => index as u64,
-
-            FieldsShape::Arbitrary { .. } => 1 + (self.fields.memory_index(index) as u64) * 2,
-        }
+        self.scalar_llvm_type_at(cx, scalar)
     }
 
     fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option<PointeeInfo> {
@@ -496,13 +513,13 @@ fn uncached_llvm_type<'a, 'tcx>(
     defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>,
 ) -> &'a Type {
     trace!("Uncached LLVM type of {:?}", layout);
-    match layout.abi {
-        Abi::Scalar(_) => bug!("handled elsewhere"),
-        Abi::Vector { ref element, count } => {
-            let element = layout.scalar_llvm_type_at(cx, element, Size::ZERO);
+    match layout.backend_repr {
+        BackendRepr::Scalar(_) => bug!("handled elsewhere"),
+        BackendRepr::Vector { element, count } => {
+            let element = layout.scalar_llvm_type_at(cx, element);
             return cx.type_vector(element, count);
         }
-        Abi::ScalarPair(..) => {
+        BackendRepr::ScalarPair(..) => {
             return cx.type_struct(
                 &[
                     layout.scalar_pair_element_llvm_type(cx, 0, false),
@@ -511,26 +528,29 @@ fn uncached_llvm_type<'a, 'tcx>(
                 false,
             );
         }
-        Abi::Uninhabited | Abi::Aggregate { .. } => {}
+        BackendRepr::Memory { .. } => {}
     }
 
     let name = match layout.ty.kind() {
         // FIXME(eddyb) producing readable type names for trait objects can result
         // in problematically distinct types due to HRTB and subtyping (see #47638).
         // ty::Dynamic(..) |
-        ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str => {
-            let mut name = with_no_trimmed_paths(|| layout.ty.to_string());
+        ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str
+            // For performance reasons we use names only when emitting LLVM IR.
+            if !cx.sess().fewer_names() =>
+        {
+            let mut name = with_no_trimmed_paths!(layout.ty.to_string());
             if let (&ty::Adt(def, _), &Variants::Single { index }) =
                 (layout.ty.kind(), &layout.variants)
             {
-                if def.is_enum() && !def.variants.is_empty() {
-                    write!(&mut name, "::{}", def.variants[index].ident).unwrap();
+                if def.is_enum() && !def.variants().is_empty() {
+                    write!(&mut name, "::{}", def.variant(index).name).unwrap();
                 }
             }
-            if let (&ty::Generator(_, _, _), &Variants::Single { index }) =
+            if let (&ty::Coroutine(_, _), &Variants::Single { index }) =
                 (layout.ty.kind(), &layout.variants)
             {
-                write!(&mut name, "::{}", ty::GeneratorSubsts::variant_name(index)).unwrap();
+                write!(&mut name, "::{}", ty::CoroutineArgs::variant_name(index)).unwrap();
             }
             Some(name)
         }
@@ -587,15 +607,17 @@ fn struct_llfields<'a, 'tcx>(
 
         assert!(target_offset >= offset);
         let padding = target_offset - offset;
-        let padding_align = prev_effective_align.min(effective_field_align);
-        assert_eq!(offset.align_to(padding_align) + padding, target_offset);
-        result.push(cx.type_padding_filler(padding, padding_align));
+        if padding != Size::ZERO {
+            let padding_align = prev_effective_align.min(effective_field_align);
+            assert_eq!(offset.align_to(padding_align) + padding, target_offset);
+            result.push(cx.type_padding_filler(padding, padding_align));
+        }
 
         result.push(field.llvm_type(cx));
         offset = target_offset + field.size;
         prev_effective_align = effective_field_align;
     }
-    if !layout.is_unsized() && field_count > 0 {
+    if layout.is_sized() && field_count > 0 {
         if offset > layout.size {
             bug!(
                 "layout: {:#?} stride: {:?} offset: {:?}",
@@ -605,10 +627,11 @@ fn struct_llfields<'a, 'tcx>(
             );
         }
         let padding = layout.size - offset;
-        let padding_align = prev_effective_align;
-        assert_eq!(offset.align_to(padding_align) + padding, layout.size);
-        result.push(cx.type_padding_filler(padding, padding_align));
-        assert_eq!(result.len(), 1 + field_count * 2);
+        if padding != Size::ZERO {
+            let padding_align = prev_effective_align;
+            assert_eq!(offset.align_to(padding_align) + padding, layout.size);
+            result.push(cx.type_padding_filler(padding, padding_align));
+        }
     } else {
     }
 
@@ -629,3 +652,17 @@ impl<'a, 'tcx> CodegenCx<'a, 'tcx> {
         (layout.size, layout.align.abi)
     }
 }
+
+impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+    fn add_type_metadata(&self, _function: Self::Function, _typeid: String) {}
+
+    fn set_type_metadata(&self, _function: Self::Function, _typeid: String) {}
+
+    fn typeid_metadata(&self, _typeid: String) -> Option<Self::Metadata> {
+        None
+    }
+
+    fn add_kcfi_type_metadata(&self, _function: Self::Function, _typeid: u32) {}
+
+    fn set_kcfi_type_metadata(&self, _function: Self::Function, _typeid: u32) {}
+}
diff --git a/examples/cuda/cpu/path_tracer/Cargo.toml b/examples/cuda/cpu/path_tracer/Cargo.toml
index 14c1987d..4eb7c4fc 100644
--- a/examples/cuda/cpu/path_tracer/Cargo.toml
+++ b/examples/cuda/cpu/path_tracer/Cargo.toml
@@ -11,12 +11,12 @@ image = "0.25.5"
 path_tracer_gpu = { path = "../../gpu/path_tracer_gpu" }
 gpu_rand = { version = "0.1", path = "../../../../crates/gpu_rand" }
 optix = { version = "0.1", path = "../../../../crates/optix" }
-glium = "0.30.2"
-glutin = "0.27.0"
-imgui = "0.8.0"
-imgui-glium-renderer = "0.8.0"
-imgui-winit-support = "0.8.0"
-rayon = "1.10"
+glium = "0.36.0"
+glutin = "0.32.2"
+imgui = "0.12.0"
+imgui-glium-renderer = "0.13.0"
+imgui-winit-support = "0.13.0"
+rayon = "1.10.0"
 sysinfo = "0.33.1"
 anyhow = "1.0.53"
 
diff --git a/examples/cuda/gpu/add_gpu/src/lib.rs b/examples/cuda/gpu/add_gpu/src/lib.rs
index 92ac4e26..42f14817 100644
--- a/examples/cuda/gpu/add_gpu/src/lib.rs
+++ b/examples/cuda/gpu/add_gpu/src/lib.rs
@@ -1,10 +1,3 @@
-#![cfg_attr(
-    target_os = "cuda",
-    no_std,
-    feature(register_attr),
-    register_attr(nvvm_internal)
-)]
-
 use cuda_std::prelude::*;
 
 #[kernel]
diff --git a/examples/cuda/gpu/path_tracer_gpu/src/lib.rs b/examples/cuda/gpu/path_tracer_gpu/src/lib.rs
index 4ebd5d80..d28fae17 100644
--- a/examples/cuda/gpu/path_tracer_gpu/src/lib.rs
+++ b/examples/cuda/gpu/path_tracer_gpu/src/lib.rs
@@ -1,9 +1,4 @@
-#![cfg_attr(
-    target_os = "cuda",
-    no_std,
-    feature(register_attr),
-    register_attr(nvvm_internal)
-)]
+#![cfg_attr(target_os = "cuda", no_std, register_attr(nvvm_internal))]
 #![allow(clippy::missing_safety_doc)]
 
 extern crate alloc;
diff --git a/guide/src/guide/getting_started.md b/guide/src/guide/getting_started.md
index fc8693e8..8372e870 100644
--- a/guide/src/guide/getting_started.md
+++ b/guide/src/guide/getting_started.md
@@ -62,7 +62,6 @@ Before we can write any GPU kernels, we must add a few directives to our `lib.rs
 #![cfg_attr(
     target_os = "cuda",
     no_std,
-    feature(register_attr),
     register_attr(nvvm_internal)
 )]
 
diff --git a/rust-toolchain b/rust-toolchain
index 4937196f..78e5dc58 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -5,5 +5,5 @@
 # to the user in the error, instead of "error: invalid channel name '[toolchain]'".
 
 [toolchain]
-channel = "nightly-2025-01-23"
+channel = "nightly-2025-03-02"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
\ No newline at end of file