From e90669c0df3edd6f5d792526b98bcbf23a4d3dde Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Fri, 11 Sep 2026 17:42:04 +0000 Subject: [PATCH 1/8] migrated cuda build to torch stable abi --- CMakeLists.txt | 4 +++ README.md | 2 +- include/deep_jit/backend/cuda/kernel.hpp | 11 ++++++-- tests/test_cuda.py | 2 ++ tests/test_cuda_proj/main.cpp | 36 ++++++++++++++++++------ 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de0a67e..3567be9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,5 +17,9 @@ include_directories( link_directories(${TORCH_INSTALL_PREFIX}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs) pybind11_add_module(_C csrc/python_api.cpp) +target_compile_definitions(_C PRIVATE + TORCH_TARGET_VERSION=0x020a000000000000 + USE_CUDA +) target_link_libraries(_C PRIVATE ${TORCH_LIBRARIES} torch_python) target_link_options(_C PRIVATE "-Wl,-Bsymbolic-functions") diff --git a/README.md b/README.md index 0786969..99b290f 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ DeepJIT handles the JIT infrastructure so that kernel libraries can focus on the | **CUDA** | NVCC compiles CUDA source to CUBIN; the CUDA Driver API loads and launches kernels. | CUDA headers 12.4+, NVCC 12.9+, and PyTorch with CUDA support. | | **Ascend** | Bisheng and ld.lld compile and link Ascend kernel source; ACL loads and launches kernels. | CANN with `bin/bisheng`, `bin/ld.lld`, and the Ascend `adv_api` headers; ACL and `torch_npu` headers and runtime. | -The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, Python, pybind11, and the dependencies for the selected backend. DeepJIT is intended to be embedded into your extension as a header-only dependency. +The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, Python, pybind11, and the dependencies for the selected backend. DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020a000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.10 as the minimum runtime version. See [Integration](#integration) for setup, [CUDA](#cuda) for GPU usage, and [Ascend](#ascend) for NPU usage. diff --git a/include/deep_jit/backend/cuda/kernel.hpp b/include/deep_jit/backend/cuda/kernel.hpp index a2ae2dd..73525a9 100644 --- a/include/deep_jit/backend/cuda/kernel.hpp +++ b/include/deep_jit/backend/cuda/kernel.hpp @@ -8,8 +8,10 @@ #include #include -#include #include +#include +#include +#include #include #include @@ -161,9 +163,14 @@ class Kernel { config.blockDimY = launch_options.block_dim->y; config.blockDimZ = launch_options.block_dim->z; config.sharedMemBytes = *launch_options.num_smem_bytes; + void* current_stream = nullptr; + if (not launch_options.stream) { + const auto device_index = torch::stable::accelerator::getCurrentDeviceIndex(); + TORCH_ERROR_CODE_CHECK(aoti_torch_get_current_cuda_stream(device_index, ¤t_stream)); + } config.hStream = launch_options.stream ? *launch_options.stream - : at::cuda::getCurrentCUDAStream().stream(); + : static_cast(current_stream); config.attrs = num_attributes == 0 ? nullptr : attributes.data(); config.numAttrs = num_attributes; DJ_CUDA_DRIVER_CHECK(driver::lazy_cuLaunchKernelEx( diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 360594a..31c4907 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -222,6 +222,7 @@ def validate_header_self_containment(temporary_dir): source_path.write_text(f'#include <{header.as_posix()}>\n', encoding='utf-8') command = [ os.environ.get('CXX', 'c++'), '-std=c++20', '-fsyntax-only', '-Werror', + '-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA', '-Wno-attributes', '-Wno-deprecated-declarations', '-Wno-missing-field-initializers', '-Wno-psabi', str(source_path), @@ -787,6 +788,7 @@ def run_worker(): sources=[str(TEST_CUDA_PROJECT / 'main.cpp')], extra_cflags=[ '-std=c++20', '-O3', '-fPIC', '-Wall', '-Wextra', '-Werror', + '-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA', '-Wno-attributes', '-Wno-missing-field-initializers', '-Wno-psabi', '-Wno-deprecated-declarations', ], diff --git a/tests/test_cuda_proj/main.cpp b/tests/test_cuda_proj/main.cpp index 4baf23e..0b741e0 100644 --- a/tests/test_cuda_proj/main.cpp +++ b/tests/test_cuda_proj/main.cpp @@ -22,8 +22,9 @@ #include #include -#include #include +#include +#include #include #include @@ -52,6 +53,23 @@ int python_api_num_initializations = 0; std::shared_ptr process_test_runtime; std::shared_ptr gil_test_runtime; +class TorchCUDAStreamGuard { +public: + TorchCUDAStreamGuard(cudaStream_t stream, const int device_index) + : device_index(device_index) { + TORCH_ERROR_CODE_CHECK(aoti_torch_get_current_cuda_stream(device_index, &previous_stream)); + TORCH_ERROR_CODE_CHECK(torch_set_current_cuda_stream(stream, device_index)); + } + + ~TorchCUDAStreamGuard() { + (void)torch_set_current_cuda_stream(previous_stream, device_index); + } + +private: + int device_index; + void* previous_stream = nullptr; +}; + const fs::path& get_test_cuda_project_dir() { static const fs::path path = [] { const auto value = std::getenv("DEEP_JIT_CUDA_TEST_SOURCE_DIR"); @@ -1819,13 +1837,14 @@ void test_runtime_launch_features(Runtime& runtime) { runtime.default_launch_options.stream = std::nullopt; runtime.default_launch_options.enable_pdl = false; - const auto current_stream = c10::cuda::getStreamFromPool(); + cudaStream_t current_stream = nullptr; + DJ_CUDA_RUNTIME_CHECK(cudaStreamCreate(¤t_stream)); cudaGraph_t graph = nullptr; cudaGraphExec_t graph_exec = nullptr; { - const c10::cuda::CUDAStreamGuard stream_guard(current_stream); - DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream.stream(), cudaStreamCaptureModeGlobal)); + const TorchCUDAStreamGuard stream_guard(current_stream, device_index); + DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream, cudaStreamCaptureModeGlobal)); runtime.launch( kernel, { .num_smem_bytes = sizeof(int), @@ -1833,12 +1852,12 @@ void test_runtime_launch_features(Runtime& runtime) { .block_dim = dim3(32, 1, 1), }, output, argument_storage, 10); - DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream.stream(), &graph)); + DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream, &graph)); } check_graph(graph); DJ_CUDA_RUNTIME_CHECK(cudaGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0)); - DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream.stream())); - DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream.stream())); + DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream)); + DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream)); check_output(43, 44); DJ_CUDA_RUNTIME_CHECK(cudaGraphExecDestroy(graph_exec)); DJ_CUDA_RUNTIME_CHECK(cudaGraphDestroy(graph)); @@ -1846,7 +1865,7 @@ void test_runtime_launch_features(Runtime& runtime) { graph = nullptr; graph_exec = nullptr; { - const c10::cuda::CUDAStreamGuard stream_guard(current_stream); + const TorchCUDAStreamGuard stream_guard(current_stream, device_index); DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal)); runtime.launch( kernel, { @@ -1866,6 +1885,7 @@ void test_runtime_launch_features(Runtime& runtime) { DJ_CUDA_RUNTIME_CHECK(cudaGraphExecDestroy(graph_exec)); DJ_CUDA_RUNTIME_CHECK(cudaGraphDestroy(graph)); + DJ_CUDA_RUNTIME_CHECK(cudaStreamDestroy(current_stream)); DJ_CUDA_RUNTIME_CHECK(cudaStreamDestroy(stream)); stream = nullptr; DJ_CUDA_RUNTIME_CHECK(cudaFree(output)); From a40ccf5e510f27ed95dd73d4bb932dcec36db6f9 Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Fri, 11 Sep 2026 19:19:17 +0000 Subject: [PATCH 2/8] updated kernel.h and main.cpp to match ehat was there before more accuratly --- include/deep_jit/backend/cuda/kernel.hpp | 16 +++++++---- tests/test_cuda_proj/main.cpp | 35 +++++++++++++++++------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/include/deep_jit/backend/cuda/kernel.hpp b/include/deep_jit/backend/cuda/kernel.hpp index 73525a9..73ceb94 100644 --- a/include/deep_jit/backend/cuda/kernel.hpp +++ b/include/deep_jit/backend/cuda/kernel.hpp @@ -31,6 +31,15 @@ inline void* kernel_arg_pointer(const T& value) { } } +// Utility to get the current CUDA stream for a given device using stable APIs. +// Returns a CUstream for use with the CUDA Driver API. +inline CUstream get_current_cuda_stream(const int32_t device_index) { + void* stream_ptr = nullptr; + TORCH_ERROR_CODE_CHECK( + aoti_torch_get_current_cuda_stream(device_index, &stream_ptr)); + return static_cast(stream_ptr); +} + // Immutable CUDA kernel handles with shared ownership. Driver resources are // unloaded when the last shared owner is destroyed. class Kernel { @@ -163,14 +172,9 @@ class Kernel { config.blockDimY = launch_options.block_dim->y; config.blockDimZ = launch_options.block_dim->z; config.sharedMemBytes = *launch_options.num_smem_bytes; - void* current_stream = nullptr; - if (not launch_options.stream) { - const auto device_index = torch::stable::accelerator::getCurrentDeviceIndex(); - TORCH_ERROR_CODE_CHECK(aoti_torch_get_current_cuda_stream(device_index, ¤t_stream)); - } config.hStream = launch_options.stream ? *launch_options.stream - : static_cast(current_stream); + : get_current_cuda_stream(torch::stable::accelerator::getCurrentDeviceIndex()); config.attrs = num_attributes == 0 ? nullptr : attributes.data(); config.numAttrs = num_attributes; DJ_CUDA_DRIVER_CHECK(driver::lazy_cuLaunchKernelEx( diff --git a/tests/test_cuda_proj/main.cpp b/tests/test_cuda_proj/main.cpp index 0b741e0..74c6b87 100644 --- a/tests/test_cuda_proj/main.cpp +++ b/tests/test_cuda_proj/main.cpp @@ -53,21 +53,38 @@ int python_api_num_initializations = 0; std::shared_ptr process_test_runtime; std::shared_ptr gil_test_runtime; +cudaStream_t get_stream_from_pool(const int32_t device_index) { + void* stream = nullptr; + TORCH_ERROR_CODE_CHECK( + torch_get_cuda_stream_from_pool(false, device_index, &stream)); + return static_cast(stream); +} + class TorchCUDAStreamGuard { public: - TorchCUDAStreamGuard(cudaStream_t stream, const int device_index) - : device_index(device_index) { - TORCH_ERROR_CODE_CHECK(aoti_torch_get_current_cuda_stream(device_index, &previous_stream)); - TORCH_ERROR_CODE_CHECK(torch_set_current_cuda_stream(stream, device_index)); + TorchCUDAStreamGuard() = delete; + + explicit TorchCUDAStreamGuard(cudaStream_t stream, int32_t device_index) { + TORCH_ERROR_CODE_CHECK( + aoti_torch_create_cuda_stream_guard( + static_cast(stream), device_index, &guard_)); } ~TorchCUDAStreamGuard() { - (void)torch_set_current_cuda_stream(previous_stream, device_index); + if (guard_) + (void)aoti_torch_delete_cuda_stream_guard(guard_); } + // Match c10::cuda::CUDAStreamGuard: copying is disallowed because the + // guard has unique ownership, and moving is disallowed because an RAII + // stream guard has no uninitialized state to leave behind. + TorchCUDAStreamGuard(const TorchCUDAStreamGuard&) = delete; + TorchCUDAStreamGuard& operator=(const TorchCUDAStreamGuard&) = delete; + TorchCUDAStreamGuard(TorchCUDAStreamGuard&&) = delete; + TorchCUDAStreamGuard& operator=(TorchCUDAStreamGuard&&) = delete; + private: - int device_index; - void* previous_stream = nullptr; + CUDAStreamGuardHandle guard_ = nullptr; }; const fs::path& get_test_cuda_project_dir() { @@ -1837,8 +1854,7 @@ void test_runtime_launch_features(Runtime& runtime) { runtime.default_launch_options.stream = std::nullopt; runtime.default_launch_options.enable_pdl = false; - cudaStream_t current_stream = nullptr; - DJ_CUDA_RUNTIME_CHECK(cudaStreamCreate(¤t_stream)); + const auto current_stream = get_stream_from_pool(device_index); cudaGraph_t graph = nullptr; cudaGraphExec_t graph_exec = nullptr; @@ -1885,7 +1901,6 @@ void test_runtime_launch_features(Runtime& runtime) { DJ_CUDA_RUNTIME_CHECK(cudaGraphExecDestroy(graph_exec)); DJ_CUDA_RUNTIME_CHECK(cudaGraphDestroy(graph)); - DJ_CUDA_RUNTIME_CHECK(cudaStreamDestroy(current_stream)); DJ_CUDA_RUNTIME_CHECK(cudaStreamDestroy(stream)); stream = nullptr; DJ_CUDA_RUNTIME_CHECK(cudaFree(output)); From e0903812c37aef466b2d5b7e3e0bd8d58c57fa9f Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Tue, 15 Sep 2026 12:26:01 +0000 Subject: [PATCH 3/8] updated TORCH_TARGET_VERSION to 2.13 and used nativeHandle() to get cuda stream instead of C shim directly --- README.md | 2 +- include/deep_jit/backend/cuda/kernel.hpp | 6 ++---- tests/test_cuda.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 99b290f..2bf1575 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ DeepJIT handles the JIT infrastructure so that kernel libraries can focus on the | **CUDA** | NVCC compiles CUDA source to CUBIN; the CUDA Driver API loads and launches kernels. | CUDA headers 12.4+, NVCC 12.9+, and PyTorch with CUDA support. | | **Ascend** | Bisheng and ld.lld compile and link Ascend kernel source; ACL loads and launches kernels. | CANN with `bin/bisheng`, `bin/ld.lld`, and the Ascend `adv_api` headers; ACL and `torch_npu` headers and runtime. | -The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, Python, pybind11, and the dependencies for the selected backend. DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020a000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.10 as the minimum runtime version. +The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, Python, pybind11, and the dependencies for the selected backend. DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020d000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.13 as the minimum runtime version. See [Integration](#integration) for setup, [CUDA](#cuda) for GPU usage, and [Ascend](#ascend) for NPU usage. diff --git a/include/deep_jit/backend/cuda/kernel.hpp b/include/deep_jit/backend/cuda/kernel.hpp index 73ceb94..415671f 100644 --- a/include/deep_jit/backend/cuda/kernel.hpp +++ b/include/deep_jit/backend/cuda/kernel.hpp @@ -34,10 +34,8 @@ inline void* kernel_arg_pointer(const T& value) { // Utility to get the current CUDA stream for a given device using stable APIs. // Returns a CUstream for use with the CUDA Driver API. inline CUstream get_current_cuda_stream(const int32_t device_index) { - void* stream_ptr = nullptr; - TORCH_ERROR_CODE_CHECK( - aoti_torch_get_current_cuda_stream(device_index, &stream_ptr)); - return static_cast(stream_ptr); + auto stream = torch::stable::accelerator::getCurrentStream(device_index); + return static_cast(stream.nativeHandle()); } // Immutable CUDA kernel handles with shared ownership. Driver resources are diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 31c4907..d95daee 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -222,7 +222,7 @@ def validate_header_self_containment(temporary_dir): source_path.write_text(f'#include <{header.as_posix()}>\n', encoding='utf-8') command = [ os.environ.get('CXX', 'c++'), '-std=c++20', '-fsyntax-only', '-Werror', - '-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA', + '-DTORCH_TARGET_VERSION=0x020d000000000000', '-DUSE_CUDA', '-Wno-attributes', '-Wno-deprecated-declarations', '-Wno-missing-field-initializers', '-Wno-psabi', str(source_path), From 866260433b37fa977b3de99dcf9db76d885a0cc1 Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Tue, 15 Sep 2026 14:36:03 -0400 Subject: [PATCH 4/8] Update tests/test_cuda.py Co-authored-by: Mehmet Oguz Derin --- tests/test_cuda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cuda.py b/tests/test_cuda.py index d95daee..a956f15 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -788,7 +788,7 @@ def run_worker(): sources=[str(TEST_CUDA_PROJECT / 'main.cpp')], extra_cflags=[ '-std=c++20', '-O3', '-fPIC', '-Wall', '-Wextra', '-Werror', - '-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA', + '-DTORCH_TARGET_VERSION=0x020d000000000000', '-DUSE_CUDA', '-Wno-attributes', '-Wno-missing-field-initializers', '-Wno-psabi', '-Wno-deprecated-declarations', ], From c2e8a2e57c1290d8ee131de3bc296a862a34ee2c Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Tue, 15 Sep 2026 14:36:58 -0400 Subject: [PATCH 5/8] Update CMakeLists.txt Co-authored-by: Mehmet Oguz Derin --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3567be9..680eddd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ link_directories(${TORCH_INSTALL_PREFIX}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ${CU pybind11_add_module(_C csrc/python_api.cpp) target_compile_definitions(_C PRIVATE - TORCH_TARGET_VERSION=0x020a000000000000 + TORCH_TARGET_VERSION=0x020d000000000000 USE_CUDA ) target_link_libraries(_C PRIVATE ${TORCH_LIBRARIES} torch_python) From e4f3e5b99761605ff182b34f4de10792348c738e Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Tue, 22 Sep 2026 18:42:41 +0000 Subject: [PATCH 6/8] added a macro to disable pybind and the gil release. A no-op GilScopedRelease used when this macro is set Signed-off-by: Chris Leonard --- README.md | 9 ++++++++- include/deep_jit/utils/gil.hpp | 24 +++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2bf1575..f3b8eb6 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ DeepJIT handles the JIT infrastructure so that kernel libraries can focus on the | **CUDA** | NVCC compiles CUDA source to CUBIN; the CUDA Driver API loads and launches kernels. | CUDA headers 12.4+, NVCC 12.9+, and PyTorch with CUDA support. | | **Ascend** | Bisheng and ld.lld compile and link Ascend kernel source; ACL loads and launches kernels. | CANN with `bin/bisheng`, `bin/ld.lld`, and the Ascend `adv_api` headers; ACL and `torch_npu` headers and runtime. | -The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, Python, pybind11, and the dependencies for the selected backend. DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020d000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.13 as the minimum runtime version. +The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, and the dependencies for the selected backend. The default GIL support also requires Python and pybind11; consumers that manage the GIL themselves can disable it as described under [Integration](#integration). DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020d000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.13 as the minimum runtime version. See [Integration](#integration) for setup, [CUDA](#cuda) for GPU usage, and [Ascend](#ascend) for NPU usage. @@ -66,6 +66,13 @@ The root `CMakeLists.txt` is for debugging and IDE indexing. Integrate the heade ## Integration +GIL management is enabled by default and includes pybind11. Consumers that manage +the GIL themselves, such as kernels called through `torch.ops`, can compile with +`-DDJ_DISABLE_GIL=1` to make `deep_jit::GilScopedRelease` a no-op without including +pybind11 or the Python C API from this helper. In CMake, use +`target_compile_definitions(my_target PRIVATE DJ_DISABLE_GIL=1)`. Set the macro +consistently for every source file compiled into the consumer target. + Add `DeepJIT/include` to the include path of the host target, then include: ```cmake diff --git a/include/deep_jit/utils/gil.hpp b/include/deep_jit/utils/gil.hpp index dc47608..2e39da0 100644 --- a/include/deep_jit/utils/gil.hpp +++ b/include/deep_jit/utils/gil.hpp @@ -1,7 +1,11 @@ #pragma once -#include +#ifndef DJ_DISABLE_GIL +#define DJ_DISABLE_GIL 0 +#endif +#if !DJ_DISABLE_GIL +#include #include namespace deep_jit { @@ -22,3 +26,21 @@ class GilScopedRelease { }; } // namespace deep_jit + +#else + +namespace deep_jit { + +class GilScopedRelease { +public: + GilScopedRelease() = default; + + GilScopedRelease(const GilScopedRelease&) = delete; + GilScopedRelease& operator=(const GilScopedRelease&) = delete; + GilScopedRelease(GilScopedRelease&&) = delete; + GilScopedRelease& operator=(GilScopedRelease&&) = delete; +}; + +} // namespace deep_jit + +#endif From 81ca29e1ba1ee1f9be04995a285beb8f5b0c0396 Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Tue, 22 Sep 2026 19:28:03 +0000 Subject: [PATCH 7/8] mark no-op GIL guards as maybe_unused. Keep DJ_DISABLE_GIL=1 builds clean under -Werror across CUDA, Ascend, and the shared runtime. Signed-off-by: Chris Leonard --- include/deep_jit/backend/ascend/backend.hpp | 2 +- include/deep_jit/backend/ascend/kernel.hpp | 4 ++-- include/deep_jit/backend/cuda/backend.hpp | 2 +- include/deep_jit/backend/cuda/kernel.hpp | 4 ++-- include/deep_jit/runtime/runtime.hpp | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/deep_jit/backend/ascend/backend.hpp b/include/deep_jit/backend/ascend/backend.hpp index d98ddb6..6c5aec4 100644 --- a/include/deep_jit/backend/ascend/backend.hpp +++ b/include/deep_jit/backend/ascend/backend.hpp @@ -76,7 +76,7 @@ class Ascend { const Config& config, const CompilerOptions& options) const { // Release GIL to let other Python threads run - GilScopedRelease gil_release; + [[maybe_unused]] GilScopedRelease gil_release; // Paths dir = std::filesystem::absolute(dir).lexically_normal(); diff --git a/include/deep_jit/backend/ascend/kernel.hpp b/include/deep_jit/backend/ascend/kernel.hpp index 018767b..b8e03b8 100644 --- a/include/deep_jit/backend/ascend/kernel.hpp +++ b/include/deep_jit/backend/ascend/kernel.hpp @@ -123,7 +123,7 @@ class Kernel { static std::shared_ptr load(const std::filesystem::path& dir, const Env& env) { // Release GIL to let other Python threads run - GilScopedRelease gil_release; + [[maybe_unused]] GilScopedRelease gil_release; // Check existence const auto binary_path = dir / "kernel.o"; @@ -159,7 +159,7 @@ class Kernel { template void launch(const LaunchOptions& launch_options, const Args&... args) const { // Release GIL to let other Python threads run - GilScopedRelease gil_release; + [[maybe_unused]] GilScopedRelease gil_release; // Checks DJ_HOST_ASSERT(kernel_handle != nullptr, "kernel must be loaded before launch"); diff --git a/include/deep_jit/backend/cuda/backend.hpp b/include/deep_jit/backend/cuda/backend.hpp index 81dfedd..e56eae1 100644 --- a/include/deep_jit/backend/cuda/backend.hpp +++ b/include/deep_jit/backend/cuda/backend.hpp @@ -80,7 +80,7 @@ class CUDA { const Config& config, const CompilerOptions& options) const { // Release GIL to let other Python threads run - GilScopedRelease gil_release; + [[maybe_unused]] GilScopedRelease gil_release; // Paths dir = std::filesystem::absolute(dir).lexically_normal(); diff --git a/include/deep_jit/backend/cuda/kernel.hpp b/include/deep_jit/backend/cuda/kernel.hpp index 415671f..3e9c5cf 100644 --- a/include/deep_jit/backend/cuda/kernel.hpp +++ b/include/deep_jit/backend/cuda/kernel.hpp @@ -56,7 +56,7 @@ class Kernel { static std::shared_ptr load(const std::filesystem::path& dir, const Env& env) { // Release GIL to let other Python threads run - GilScopedRelease gil_release; + [[maybe_unused]] GilScopedRelease gil_release; // Check existence const auto cubin_path = dir / "kernel.cubin"; @@ -100,7 +100,7 @@ class Kernel { template void launch(const LaunchOptions& launch_options, const Args&... args) const { // Release GIL to let other Python threads run - GilScopedRelease gil_release; + [[maybe_unused]] GilScopedRelease gil_release; // Checks DJ_HOST_ASSERT(kernel_handle != nullptr, "kernel must be loaded before launch"); diff --git a/include/deep_jit/runtime/runtime.hpp b/include/deep_jit/runtime/runtime.hpp index 8ca176c..1c15175 100644 --- a/include/deep_jit/runtime/runtime.hpp +++ b/include/deep_jit/runtime/runtime.hpp @@ -85,7 +85,7 @@ class Runtime { backend.compile(source, entry.path, env, config, options); // Publish - GilScopedRelease gil_release; + [[maybe_unused]] GilScopedRelease gil_release; return entry.commit(); } From bad7a43e852cb478e07a2170e474d4b280f34064 Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Mon, 28 Sep 2026 14:29:28 +0000 Subject: [PATCH 8/8] Move Torch stable shim helpers into shared header Signed-off-by: Chris Leonard --- CMakeLists.txt | 2 +- README.md | 2 +- include/deep_jit/backend/cuda/kernel.hpp | 11 +-- .../backend/cuda/stable_torch_utils.h | 70 +++++++++++++++++++ tests/test_cuda.py | 4 +- tests/test_cuda_proj/main.cpp | 39 +---------- 6 files changed, 78 insertions(+), 50 deletions(-) create mode 100644 include/deep_jit/backend/cuda/stable_torch_utils.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 680eddd..3567be9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ link_directories(${TORCH_INSTALL_PREFIX}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ${CU pybind11_add_module(_C csrc/python_api.cpp) target_compile_definitions(_C PRIVATE - TORCH_TARGET_VERSION=0x020d000000000000 + TORCH_TARGET_VERSION=0x020a000000000000 USE_CUDA ) target_link_libraries(_C PRIVATE ${TORCH_LIBRARIES} torch_python) diff --git a/README.md b/README.md index f3b8eb6..02cc44e 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ DeepJIT handles the JIT infrastructure so that kernel libraries can focus on the | **CUDA** | NVCC compiles CUDA source to CUBIN; the CUDA Driver API loads and launches kernels. | CUDA headers 12.4+, NVCC 12.9+, and PyTorch with CUDA support. | | **Ascend** | Bisheng and ld.lld compile and link Ascend kernel source; ACL loads and launches kernels. | CANN with `bin/bisheng`, `bin/ld.lld`, and the Ascend `adv_api` headers; ACL and `torch_npu` headers and runtime. | -The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, and the dependencies for the selected backend. The default GIL support also requires Python and pybind11; consumers that manage the GIL themselves can disable it as described under [Integration](#integration). DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020d000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.13 as the minimum runtime version. +The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, and the dependencies for the selected backend. The default GIL support also requires Python and pybind11; consumers that manage the GIL themselves can disable it as described under [Integration](#integration). DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020a000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.10 as the minimum runtime version. See [Integration](#integration) for setup, [CUDA](#cuda) for GPU usage, and [Ascend](#ascend) for NPU usage. diff --git a/include/deep_jit/backend/cuda/kernel.hpp b/include/deep_jit/backend/cuda/kernel.hpp index 3e9c5cf..915acf5 100644 --- a/include/deep_jit/backend/cuda/kernel.hpp +++ b/include/deep_jit/backend/cuda/kernel.hpp @@ -9,12 +9,10 @@ #include #include -#include -#include -#include #include #include +#include #include #include #include @@ -31,13 +29,6 @@ inline void* kernel_arg_pointer(const T& value) { } } -// Utility to get the current CUDA stream for a given device using stable APIs. -// Returns a CUstream for use with the CUDA Driver API. -inline CUstream get_current_cuda_stream(const int32_t device_index) { - auto stream = torch::stable::accelerator::getCurrentStream(device_index); - return static_cast(stream.nativeHandle()); -} - // Immutable CUDA kernel handles with shared ownership. Driver resources are // unloaded when the last shared owner is destroyed. class Kernel { diff --git a/include/deep_jit/backend/cuda/stable_torch_utils.h b/include/deep_jit/backend/cuda/stable_torch_utils.h new file mode 100644 index 0000000..2c1b438 --- /dev/null +++ b/include/deep_jit/backend/cuda/stable_torch_utils.h @@ -0,0 +1,70 @@ +#pragma once + +#include + +#include +#include + +// Expose the CUDA declarations in Torch's shims without adding USE_CUDA to +// the rest of the including translation unit. +#ifndef USE_CUDA +#define DEEP_JIT_UNDEFINE_USE_CUDA +#define USE_CUDA +#endif +#include +#include +#include +#ifdef DEEP_JIT_UNDEFINE_USE_CUDA +#undef USE_CUDA +#undef DEEP_JIT_UNDEFINE_USE_CUDA +#endif + +namespace deep_jit::cuda { + +// Returns the current CUDA stream for use with the CUDA Driver API. +inline CUstream get_current_cuda_stream(const int32_t device_index) { + void* stream_ptr = nullptr; + TORCH_ERROR_CODE_CHECK( + aoti_torch_get_current_cuda_stream(device_index, &stream_ptr)); + return static_cast(stream_ptr); +} + +// Upstream torch plans to add stable API's for get_stream_from_pool and TorchCUDAStreamGuard. +// Once they are added and TORCH_TARGET_VERSION catches up, this file can be deleted +// and the APIs can be called directly. + +inline cudaStream_t get_stream_from_pool(const int32_t device_index) { + void* stream = nullptr; + TORCH_ERROR_CODE_CHECK( + torch_get_cuda_stream_from_pool(false, device_index, &stream)); + return static_cast(stream); +} + +class TorchCUDAStreamGuard { +public: + TorchCUDAStreamGuard() = delete; + + explicit TorchCUDAStreamGuard(cudaStream_t stream, int32_t device_index) { + TORCH_ERROR_CODE_CHECK( + aoti_torch_create_cuda_stream_guard( + static_cast(stream), device_index, &guard_)); + } + + ~TorchCUDAStreamGuard() { + if (guard_) + (void)aoti_torch_delete_cuda_stream_guard(guard_); + } + + // Match c10::cuda::CUDAStreamGuard: copying is disallowed because the + // guard has unique ownership, and moving is disallowed because an RAII + // stream guard has no uninitialized state to leave behind. + TorchCUDAStreamGuard(const TorchCUDAStreamGuard&) = delete; + TorchCUDAStreamGuard& operator=(const TorchCUDAStreamGuard&) = delete; + TorchCUDAStreamGuard(TorchCUDAStreamGuard&&) = delete; + TorchCUDAStreamGuard& operator=(TorchCUDAStreamGuard&&) = delete; + +private: + CUDAStreamGuardHandle guard_ = nullptr; +}; + +} // namespace deep_jit::cuda diff --git a/tests/test_cuda.py b/tests/test_cuda.py index a956f15..31c4907 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -222,7 +222,7 @@ def validate_header_self_containment(temporary_dir): source_path.write_text(f'#include <{header.as_posix()}>\n', encoding='utf-8') command = [ os.environ.get('CXX', 'c++'), '-std=c++20', '-fsyntax-only', '-Werror', - '-DTORCH_TARGET_VERSION=0x020d000000000000', '-DUSE_CUDA', + '-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA', '-Wno-attributes', '-Wno-deprecated-declarations', '-Wno-missing-field-initializers', '-Wno-psabi', str(source_path), @@ -788,7 +788,7 @@ def run_worker(): sources=[str(TEST_CUDA_PROJECT / 'main.cpp')], extra_cflags=[ '-std=c++20', '-O3', '-fPIC', '-Wall', '-Wextra', '-Werror', - '-DTORCH_TARGET_VERSION=0x020d000000000000', '-DUSE_CUDA', + '-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA', '-Wno-attributes', '-Wno-missing-field-initializers', '-Wno-psabi', '-Wno-deprecated-declarations', ], diff --git a/tests/test_cuda_proj/main.cpp b/tests/test_cuda_proj/main.cpp index 74c6b87..c344809 100644 --- a/tests/test_cuda_proj/main.cpp +++ b/tests/test_cuda_proj/main.cpp @@ -23,11 +23,10 @@ #include #include -#include -#include #include #include +#include #include #include #include @@ -46,6 +45,8 @@ namespace { using Runtime = deep_jit::Runtime; using CompilerOptions = deep_jit::cuda::CompilerOptions; using LaunchOptions = deep_jit::cuda::LaunchOptions; +using TorchCUDAStreamGuard = deep_jit::cuda::TorchCUDAStreamGuard; +using deep_jit::cuda::get_stream_from_pool; namespace fs = std::filesystem; deep_jit::LazyInit python_api_jit(nullptr); @@ -53,40 +54,6 @@ int python_api_num_initializations = 0; std::shared_ptr process_test_runtime; std::shared_ptr gil_test_runtime; -cudaStream_t get_stream_from_pool(const int32_t device_index) { - void* stream = nullptr; - TORCH_ERROR_CODE_CHECK( - torch_get_cuda_stream_from_pool(false, device_index, &stream)); - return static_cast(stream); -} - -class TorchCUDAStreamGuard { -public: - TorchCUDAStreamGuard() = delete; - - explicit TorchCUDAStreamGuard(cudaStream_t stream, int32_t device_index) { - TORCH_ERROR_CODE_CHECK( - aoti_torch_create_cuda_stream_guard( - static_cast(stream), device_index, &guard_)); - } - - ~TorchCUDAStreamGuard() { - if (guard_) - (void)aoti_torch_delete_cuda_stream_guard(guard_); - } - - // Match c10::cuda::CUDAStreamGuard: copying is disallowed because the - // guard has unique ownership, and moving is disallowed because an RAII - // stream guard has no uninitialized state to leave behind. - TorchCUDAStreamGuard(const TorchCUDAStreamGuard&) = delete; - TorchCUDAStreamGuard& operator=(const TorchCUDAStreamGuard&) = delete; - TorchCUDAStreamGuard(TorchCUDAStreamGuard&&) = delete; - TorchCUDAStreamGuard& operator=(TorchCUDAStreamGuard&&) = delete; - -private: - CUDAStreamGuardHandle guard_ = nullptr; -}; - const fs::path& get_test_cuda_project_dir() { static const fs::path path = [] { const auto value = std::getenv("DEEP_JIT_CUDA_TEST_SOURCE_DIR");