Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,9 @@ include_directories(
link_directories(${TORCH_INSTALL_PREFIX}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs)

pybind11_add_module(_C csrc/python_api.cpp)
target_compile_definitions(_C PRIVATE
TORCH_TARGET_VERSION=0x020d000000000000

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be better to target 2.10 as minimal requirement as it's feasible. For ascend it's ok to target 2.13 if it's necessary.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

To move to torch 2.10, we needed to replace

auto stream = torch::stable::accelerator::getCurrentStream(device_index);
return static_cast<CUstream>(stream.nativeHandle());

with

void* stream_ptr = nullptr;
TORCH_ERROR_CODE_CHECK(
    aoti_torch_get_current_cuda_stream(device_index, &stream_ptr));
return static_cast<CUstream>(stream_ptr);

which requires the macro USE_CUDA to be defined since that method is guarded in PyTorch, so I added this guard in the new include/deep_jit/backend/cuda/stable_torch_utils.h‎ file for it to work

#ifndef USE_CUDA
#define DEEP_JIT_UNDEFINE_USE_CUDA
#define USE_CUDA
#endif
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/c/shim.h>
#include <torch/headeronly/util/shim_utils.h>
#ifdef DEEP_JIT_UNDEFINE_USE_CUDA
#undef USE_CUDA
#undef DEEP_JIT_UNDEFINE_USE_CUDA
#endif

The final #ifdef block is so the rest of the build is not polluted with the USE_CUDA macro.

USE_CUDA
)
target_link_libraries(_C PRIVATE ${TORCH_LIBRARIES} torch_python)
target_link_options(_C PRIVATE "-Wl,-Bsymbolic-functions")
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ DeepJIT handles the JIT infrastructure so that kernel libraries can focus on the
| **CUDA** | NVCC compiles CUDA source to CUBIN; the CUDA Driver API loads and launches kernels. | CUDA headers 12.4+, NVCC 12.9+, and PyTorch with CUDA support. |
| **Ascend** | Bisheng and ld.lld compile and link Ascend kernel source; ACL loads and launches kernels. | CANN with `bin/bisheng`, `bin/ld.lld`, and the Ascend `adv_api` headers; ACL and `torch_npu` headers and runtime. |

The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, Python, pybind11, and the dependencies for the selected backend. DeepJIT is intended to be embedded into your extension as a header-only dependency.
The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, and the dependencies for the selected backend. The default GIL support also requires Python and pybind11; consumers that manage the GIL themselves can disable it as described under [Integration](#integration). DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020d000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.13 as the minimum runtime version.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cleonard530 if we use the old cuda stream shim, what's the min runtime version we can target?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could target 2.10 then (TORCH_TARGET_VERSION=0x020a000000000000)


See [Integration](#integration) for setup, [CUDA](#cuda) for GPU usage, and [Ascend](#ascend) for NPU usage.

Expand Down Expand Up @@ -66,6 +66,13 @@ The root `CMakeLists.txt` is for debugging and IDE indexing. Integrate the heade

## Integration

GIL management is enabled by default and includes pybind11. Consumers that manage
the GIL themselves, such as kernels called through `torch.ops`, can compile with
`-DDJ_DISABLE_GIL=1` to make `deep_jit::GilScopedRelease` a no-op without including
pybind11 or the Python C API from this helper. In CMake, use
`target_compile_definitions(my_target PRIVATE DJ_DISABLE_GIL=1)`. Set the macro
consistently for every source file compiled into the consumer target.

Add `DeepJIT/include` to the include path of the host target, then include:

```cmake
Expand Down
2 changes: 1 addition & 1 deletion include/deep_jit/backend/ascend/backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class Ascend {
const Config& config,
const CompilerOptions& options) const {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Paths
dir = std::filesystem::absolute(dir).lexically_normal();
Expand Down
4 changes: 2 additions & 2 deletions include/deep_jit/backend/ascend/kernel.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ class Kernel {

static std::shared_ptr<Kernel> load(const std::filesystem::path& dir, const Env& env) {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Check existence
const auto binary_path = dir / "kernel.o";
Expand Down Expand Up @@ -159,7 +159,7 @@ class Kernel {
template <typename... Args>
void launch(const LaunchOptions& launch_options, const Args&... args) const {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Checks
DJ_HOST_ASSERT(kernel_handle != nullptr, "kernel must be loaded before launch");
Expand Down
2 changes: 1 addition & 1 deletion include/deep_jit/backend/cuda/backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ class CUDA {
const Config& config,
const CompilerOptions& options) const {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Paths
dir = std::filesystem::absolute(dir).lexically_normal();
Expand Down
17 changes: 13 additions & 4 deletions include/deep_jit/backend/cuda/kernel.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@
#include <memory>
#include <type_traits>

#include <ATen/cuda/CUDAContext.h>
#include <cuda.h>
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/c/shim.h>
#include <torch/headeronly/util/shim_utils.h>

#include <deep_jit/backend/cuda/driver.hpp>
#include <deep_jit/backend/cuda/options.hpp>
Expand All @@ -29,6 +31,13 @@ inline void* kernel_arg_pointer(const T& value) {
}
}

// Utility to get the current CUDA stream for a given device using stable APIs.
// Returns a CUstream for use with the CUDA Driver API.
inline CUstream get_current_cuda_stream(const int32_t device_index) {
auto stream = torch::stable::accelerator::getCurrentStream(device_index);
return static_cast<CUstream>(stream.nativeHandle());
}

// Immutable CUDA kernel handles with shared ownership. Driver resources are
// unloaded when the last shared owner is destroyed.
class Kernel {
Expand All @@ -47,7 +56,7 @@ class Kernel {

static std::shared_ptr<Kernel> load(const std::filesystem::path& dir, const Env& env) {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Check existence
const auto cubin_path = dir / "kernel.cubin";
Expand Down Expand Up @@ -91,7 +100,7 @@ class Kernel {
template <typename... Args>
void launch(const LaunchOptions& launch_options, const Args&... args) const {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Checks
DJ_HOST_ASSERT(kernel_handle != nullptr, "kernel must be loaded before launch");
Expand Down Expand Up @@ -163,7 +172,7 @@ class Kernel {
config.sharedMemBytes = *launch_options.num_smem_bytes;
config.hStream = launch_options.stream
? *launch_options.stream
: at::cuda::getCurrentCUDAStream().stream();
: get_current_cuda_stream(torch::stable::accelerator::getCurrentDeviceIndex());
config.attrs = num_attributes == 0 ? nullptr : attributes.data();
config.numAttrs = num_attributes;
DJ_CUDA_DRIVER_CHECK(driver::lazy_cuLaunchKernelEx(
Expand Down
2 changes: 1 addition & 1 deletion include/deep_jit/runtime/runtime.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ class Runtime {
backend.compile(source, entry.path, env, config, options);

// Publish
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;
return entry.commit();
}

Expand Down
24 changes: 23 additions & 1 deletion include/deep_jit/utils/gil.hpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
#pragma once

#include <optional>
#ifndef DJ_DISABLE_GIL
#define DJ_DISABLE_GIL 0
#endif

#if !DJ_DISABLE_GIL
#include <optional>
#include <pybind11/pybind11.h>

namespace deep_jit {
Expand All @@ -22,3 +26,21 @@ class GilScopedRelease {
};

} // namespace deep_jit

#else

namespace deep_jit {

class GilScopedRelease {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not very familiar with the nogil support of pytorch upstream, would you mind elaborate a little bit? Does this mean that DeepJIT itself supports nogil? Or downstream libraries (e.g. deepgemm / deepep) needs extra changes.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When kernels are registered with TORCH_LIBRARY, the operator is registered through PyTorch's dispatcher mechanism (basically becomes just like any other PyTorch op) and can be invoked from Python through torch.ops.<op_name>. When a PyTorch operator is invoked from Python, PyTorch releases the GIL before executing the operator through the dispatcher, so by the time it reaches GilScopedRelease gil_release here, the GIL has already been released and that guard is effectively a no-op. This means that DeepJIT doesn't need to explicitly release the GIL for downstream operators that are registered through TORCH_LIBRARY/STABLE_TORCH_LIBRARY.

Currently, DeepGEMM dispatches through pybind, so the explicit GIL release is still required there. We are working on moving DeepGEMM's registration from pybind to TORCH_LIBRARY (see vllm-project/DeepGEMM#15 and deepseek-ai/DeepGEMM#393). We need that migration to happen before we can remove the explicit GIL release from this path, which is why we added a macro flag DJ_DISABLE_GIL rather than removing the release outright. This allows existing downstream users that still dispatch through pybind to continue working correctly while we migrate.

I'm not as familiar with DeepEP, so if it still relies on pybind or otherwise requires the GIL to be released at this point, that flag would need to remain for it as well, or any other downstream task that aren't registered through TORCH_LIBRARY.

Once the downstream libraries have all moved from pybind to TORCH_LIBRARY/STABLE_TORCH_LIBRARY, and assuming they don't otherwise interact with Python while executing their kernels, we should be able to remove the explicit GilScopedRelease entirely.

Let me know if you have any other questions or need anything clarified. Thanks!

public:
GilScopedRelease() = default;

GilScopedRelease(const GilScopedRelease&) = delete;
GilScopedRelease& operator=(const GilScopedRelease&) = delete;
GilScopedRelease(GilScopedRelease&&) = delete;
GilScopedRelease& operator=(GilScopedRelease&&) = delete;
};

} // namespace deep_jit

#endif
2 changes: 2 additions & 0 deletions tests/test_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ def validate_header_self_containment(temporary_dir):
source_path.write_text(f'#include <{header.as_posix()}>\n', encoding='utf-8')
command = [
os.environ.get('CXX', 'c++'), '-std=c++20', '-fsyntax-only', '-Werror',
'-DTORCH_TARGET_VERSION=0x020d000000000000', '-DUSE_CUDA',
'-Wno-attributes', '-Wno-deprecated-declarations',
'-Wno-missing-field-initializers', '-Wno-psabi',
str(source_path),
Expand Down Expand Up @@ -787,6 +788,7 @@ def run_worker():
sources=[str(TEST_CUDA_PROJECT / 'main.cpp')],
extra_cflags=[
'-std=c++20', '-O3', '-fPIC', '-Wall', '-Wextra', '-Werror',
'-DTORCH_TARGET_VERSION=0x020d000000000000', '-DUSE_CUDA',
'-Wno-attributes', '-Wno-missing-field-initializers',
'-Wno-psabi', '-Wno-deprecated-declarations',
],
Expand Down
51 changes: 43 additions & 8 deletions tests/test_cuda_proj/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@
#include <vector>

#include <cuda_runtime.h>
#include <c10/cuda/CUDAGuard.h>
#include <pybind11/pybind11.h>
#include <torch/csrc/stable/c/shim.h>
#include <torch/headeronly/util/shim_utils.h>
#include <unistd.h>

#include <deep_jit/backend/cuda/backend.hpp>
Expand Down Expand Up @@ -52,6 +53,40 @@ int python_api_num_initializations = 0;
std::shared_ptr<Runtime> process_test_runtime;
std::shared_ptr<Runtime> gil_test_runtime;

cudaStream_t get_stream_from_pool(const int32_t device_index) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If i'm not mistaken, upstream torch is planning to add stable api for these two, so maybe we could move them to a separate util file for future removal?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we can definitely move this and similar torch functions to a util file!

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

void* stream = nullptr;
TORCH_ERROR_CODE_CHECK(
torch_get_cuda_stream_from_pool(false, device_index, &stream));

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would be good to upstream a high level C++ wrapper API for this one

return static_cast<cudaStream_t>(stream);
}

class TorchCUDAStreamGuard {
public:
TorchCUDAStreamGuard() = delete;

explicit TorchCUDAStreamGuard(cudaStream_t stream, int32_t device_index) {
TORCH_ERROR_CODE_CHECK(
aoti_torch_create_cuda_stream_guard(

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one too

static_cast<void*>(stream), device_index, &guard_));
}

~TorchCUDAStreamGuard() {
if (guard_)
(void)aoti_torch_delete_cuda_stream_guard(guard_);
}

// Match c10::cuda::CUDAStreamGuard: copying is disallowed because the
// guard has unique ownership, and moving is disallowed because an RAII
// stream guard has no uninitialized state to leave behind.
TorchCUDAStreamGuard(const TorchCUDAStreamGuard&) = delete;
TorchCUDAStreamGuard& operator=(const TorchCUDAStreamGuard&) = delete;
TorchCUDAStreamGuard(TorchCUDAStreamGuard&&) = delete;
TorchCUDAStreamGuard& operator=(TorchCUDAStreamGuard&&) = delete;

private:
CUDAStreamGuardHandle guard_ = nullptr;
};

const fs::path& get_test_cuda_project_dir() {
static const fs::path path = [] {
const auto value = std::getenv("DEEP_JIT_CUDA_TEST_SOURCE_DIR");
Expand Down Expand Up @@ -1819,34 +1854,34 @@ void test_runtime_launch_features(Runtime& runtime) {

runtime.default_launch_options.stream = std::nullopt;
runtime.default_launch_options.enable_pdl = false;
const auto current_stream = c10::cuda::getStreamFromPool();
const auto current_stream = get_stream_from_pool(device_index);

cudaGraph_t graph = nullptr;
cudaGraphExec_t graph_exec = nullptr;
{
const c10::cuda::CUDAStreamGuard stream_guard(current_stream);
DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream.stream(), cudaStreamCaptureModeGlobal));
const TorchCUDAStreamGuard stream_guard(current_stream, device_index);
DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream, cudaStreamCaptureModeGlobal));
runtime.launch(
kernel, {
.num_smem_bytes = sizeof(int),
.grid_dim = dim3(2, 1, 1),
.block_dim = dim3(32, 1, 1),
},
output, argument_storage, 10);
DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream.stream(), &graph));
DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream, &graph));
}
check_graph(graph);
DJ_CUDA_RUNTIME_CHECK(cudaGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0));
DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream.stream()));
DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream.stream()));
DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream));
DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream));
check_output(43, 44);
DJ_CUDA_RUNTIME_CHECK(cudaGraphExecDestroy(graph_exec));
DJ_CUDA_RUNTIME_CHECK(cudaGraphDestroy(graph));

graph = nullptr;
graph_exec = nullptr;
{
const c10::cuda::CUDAStreamGuard stream_guard(current_stream);
const TorchCUDAStreamGuard stream_guard(current_stream, device_index);
DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal));
runtime.launch(
kernel, {
Expand Down