Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,9 @@ include_directories(
link_directories(${TORCH_INSTALL_PREFIX}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs)

pybind11_add_module(_C csrc/python_api.cpp)
target_compile_definitions(_C PRIVATE
TORCH_TARGET_VERSION=0x020a000000000000
Comment thread
cleonard530 marked this conversation as resolved.
Resolved
USE_CUDA
)
target_link_libraries(_C PRIVATE ${TORCH_LIBRARIES} torch_python)
target_link_options(_C PRIVATE "-Wl,-Bsymbolic-functions")
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ DeepJIT handles the JIT infrastructure so that kernel libraries can focus on the
| **CUDA** | NVCC compiles CUDA source to CUBIN; the CUDA Driver API loads and launches kernels. | CUDA headers 12.4+, NVCC 12.9+, and PyTorch with CUDA support. |
| **Ascend** | Bisheng and ld.lld compile and link Ascend kernel source; ACL loads and launches kernels. | CANN with `bin/bisheng`, `bin/ld.lld`, and the Ascend `adv_api` headers; ACL and `torch_npu` headers and runtime. |

The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, Python, pybind11, and the dependencies for the selected backend. DeepJIT is intended to be embedded into your extension as a header-only dependency.
The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, Python, pybind11, and the dependencies for the selected backend. DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020a000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.10 as the minimum runtime version.

See [Integration](#integration) for setup, [CUDA](#cuda) for GPU usage, and [Ascend](#ascend) for NPU usage.

Expand Down
15 changes: 13 additions & 2 deletions include/deep_jit/backend/cuda/kernel.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@
#include <memory>
#include <type_traits>

#include <ATen/cuda/CUDAContext.h>
#include <cuda.h>
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/c/shim.h>
#include <torch/headeronly/util/shim_utils.h>

#include <deep_jit/backend/cuda/driver.hpp>
#include <deep_jit/backend/cuda/options.hpp>
Expand All @@ -29,6 +31,15 @@ inline void* kernel_arg_pointer(const T& value) {
}
}

// Utility to get the current CUDA stream for a given device using stable APIs.
// Returns a CUstream for use with the CUDA Driver API.
inline CUstream get_current_cuda_stream(const int32_t device_index) {
void* stream_ptr = nullptr;
TORCH_ERROR_CODE_CHECK(
aoti_torch_get_current_cuda_stream(device_index, &stream_ptr));

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For more recent torch, we should have a more high level better UX API for this, no?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we definitely have this for torch >= 2.13. I wasn't sure how restrictive we wanted to be here, but seeing your comment below we can update this.

return static_cast<CUstream>(stream_ptr);
}

// Immutable CUDA kernel handles with shared ownership. Driver resources are
// unloaded when the last shared owner is destroyed.
class Kernel {
Expand Down Expand Up @@ -163,7 +174,7 @@ class Kernel {
config.sharedMemBytes = *launch_options.num_smem_bytes;
config.hStream = launch_options.stream
? *launch_options.stream
: at::cuda::getCurrentCUDAStream().stream();
: get_current_cuda_stream(torch::stable::accelerator::getCurrentDeviceIndex());
config.attrs = num_attributes == 0 ? nullptr : attributes.data();
config.numAttrs = num_attributes;
DJ_CUDA_DRIVER_CHECK(driver::lazy_cuLaunchKernelEx(
Expand Down
2 changes: 2 additions & 0 deletions tests/test_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ def validate_header_self_containment(temporary_dir):
source_path.write_text(f'#include <{header.as_posix()}>\n', encoding='utf-8')
command = [
os.environ.get('CXX', 'c++'), '-std=c++20', '-fsyntax-only', '-Werror',
'-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA',

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can be 2.13

'-Wno-attributes', '-Wno-deprecated-declarations',
'-Wno-missing-field-initializers', '-Wno-psabi',
str(source_path),
Expand Down Expand Up @@ -787,6 +788,7 @@ def run_worker():
sources=[str(TEST_CUDA_PROJECT / 'main.cpp')],
extra_cflags=[
'-std=c++20', '-O3', '-fPIC', '-Wall', '-Wextra', '-Werror',
'-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA',
Comment thread
cleonard530 marked this conversation as resolved.
Resolved
'-Wno-attributes', '-Wno-missing-field-initializers',
'-Wno-psabi', '-Wno-deprecated-declarations',
],
Expand Down
51 changes: 43 additions & 8 deletions tests/test_cuda_proj/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@
#include <vector>

#include <cuda_runtime.h>
#include <c10/cuda/CUDAGuard.h>
#include <pybind11/pybind11.h>
#include <torch/csrc/stable/c/shim.h>
#include <torch/headeronly/util/shim_utils.h>
#include <unistd.h>

#include <deep_jit/backend/cuda/backend.hpp>
Expand Down Expand Up @@ -52,6 +53,40 @@ int python_api_num_initializations = 0;
std::shared_ptr<Runtime> process_test_runtime;
std::shared_ptr<Runtime> gil_test_runtime;

cudaStream_t get_stream_from_pool(const int32_t device_index) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If i'm not mistaken, upstream torch is planning to add stable api for these two, so maybe we could move them to a separate util file for future removal?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we can definitely move this and similar torch functions to a util file!

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

void* stream = nullptr;
TORCH_ERROR_CODE_CHECK(
torch_get_cuda_stream_from_pool(false, device_index, &stream));

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would be good to upstream a high level C++ wrapper API for this one

return static_cast<cudaStream_t>(stream);
}

class TorchCUDAStreamGuard {
public:
TorchCUDAStreamGuard() = delete;

explicit TorchCUDAStreamGuard(cudaStream_t stream, int32_t device_index) {
TORCH_ERROR_CODE_CHECK(
aoti_torch_create_cuda_stream_guard(

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one too

static_cast<void*>(stream), device_index, &guard_));
}

~TorchCUDAStreamGuard() {
if (guard_)
(void)aoti_torch_delete_cuda_stream_guard(guard_);
}

// Match c10::cuda::CUDAStreamGuard: copying is disallowed because the
// guard has unique ownership, and moving is disallowed because an RAII
// stream guard has no uninitialized state to leave behind.
TorchCUDAStreamGuard(const TorchCUDAStreamGuard&) = delete;
TorchCUDAStreamGuard& operator=(const TorchCUDAStreamGuard&) = delete;
TorchCUDAStreamGuard(TorchCUDAStreamGuard&&) = delete;
TorchCUDAStreamGuard& operator=(TorchCUDAStreamGuard&&) = delete;

private:
CUDAStreamGuardHandle guard_ = nullptr;
};

const fs::path& get_test_cuda_project_dir() {
static const fs::path path = [] {
const auto value = std::getenv("DEEP_JIT_CUDA_TEST_SOURCE_DIR");
Expand Down Expand Up @@ -1819,34 +1854,34 @@ void test_runtime_launch_features(Runtime& runtime) {

runtime.default_launch_options.stream = std::nullopt;
runtime.default_launch_options.enable_pdl = false;
const auto current_stream = c10::cuda::getStreamFromPool();
const auto current_stream = get_stream_from_pool(device_index);

cudaGraph_t graph = nullptr;
cudaGraphExec_t graph_exec = nullptr;
{
const c10::cuda::CUDAStreamGuard stream_guard(current_stream);
DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream.stream(), cudaStreamCaptureModeGlobal));
const TorchCUDAStreamGuard stream_guard(current_stream, device_index);
DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream, cudaStreamCaptureModeGlobal));
runtime.launch(
kernel, {
.num_smem_bytes = sizeof(int),
.grid_dim = dim3(2, 1, 1),
.block_dim = dim3(32, 1, 1),
},
output, argument_storage, 10);
DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream.stream(), &graph));
DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream, &graph));
}
check_graph(graph);
DJ_CUDA_RUNTIME_CHECK(cudaGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0));
DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream.stream()));
DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream.stream()));
DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream));
DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream));
check_output(43, 44);
DJ_CUDA_RUNTIME_CHECK(cudaGraphExecDestroy(graph_exec));
DJ_CUDA_RUNTIME_CHECK(cudaGraphDestroy(graph));

graph = nullptr;
graph_exec = nullptr;
{
const c10::cuda::CUDAStreamGuard stream_guard(current_stream);
const TorchCUDAStreamGuard stream_guard(current_stream, device_index);
DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal));
runtime.launch(
kernel, {
Expand Down