Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,9 @@ include_directories(
link_directories(${TORCH_INSTALL_PREFIX}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs)

pybind11_add_module(_C csrc/python_api.cpp)
target_compile_definitions(_C PRIVATE
TORCH_TARGET_VERSION=0x020a000000000000
USE_CUDA
)
target_link_libraries(_C PRIVATE ${TORCH_LIBRARIES} torch_python)
target_link_options(_C PRIVATE "-Wl,-Bsymbolic-functions")
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ DeepJIT handles the JIT infrastructure so that kernel libraries can focus on the
| **CUDA** | NVCC compiles CUDA source to CUBIN; the CUDA Driver API loads and launches kernels. | CUDA headers 12.4+, NVCC 12.9+, and PyTorch with CUDA support. |
| **Ascend** | Bisheng and ld.lld compile and link Ascend kernel source; ACL loads and launches kernels. | CANN with `bin/bisheng`, `bin/ld.lld`, and the Ascend `adv_api` headers; ACL and `torch_npu` headers and runtime. |

The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, Python, pybind11, and the dependencies for the selected backend. DeepJIT is intended to be embedded into your extension as a header-only dependency.
The host environment must provide Linux, a C++20 compiler and standard library with `std::format` support, and the dependencies for the selected backend. The default GIL support also requires Python and pybind11; consumers that manage the GIL themselves can disable it as described under [Integration](#integration). DeepJIT is intended to be embedded into your extension as a header-only dependency. CUDA consumers should compile with `TORCH_TARGET_VERSION=0x020a000000000000` and `USE_CUDA`; this targets PyTorch's stable ABI with PyTorch 2.10 as the minimum runtime version.

See [Integration](#integration) for setup, [CUDA](#cuda) for GPU usage, and [Ascend](#ascend) for NPU usage.

Expand Down Expand Up @@ -66,6 +66,13 @@ The root `CMakeLists.txt` is for debugging and IDE indexing. Integrate the heade

## Integration

GIL management is enabled by default and includes pybind11. Consumers that manage
the GIL themselves, such as kernels called through `torch.ops`, can compile with
`-DDJ_DISABLE_GIL=1` to make `deep_jit::GilScopedRelease` a no-op without including
pybind11 or the Python C API from this helper. In CMake, use
`target_compile_definitions(my_target PRIVATE DJ_DISABLE_GIL=1)`. Set the macro
consistently for every source file compiled into the consumer target.

Add `DeepJIT/include` to the include path of the host target, then include:

```cmake
Expand Down
2 changes: 1 addition & 1 deletion include/deep_jit/backend/ascend/backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class Ascend {
const Config& config,
const CompilerOptions& options) const {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Paths
dir = std::filesystem::absolute(dir).lexically_normal();
Expand Down
4 changes: 2 additions & 2 deletions include/deep_jit/backend/ascend/kernel.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ class Kernel {

static std::shared_ptr<Kernel> load(const std::filesystem::path& dir, const Env& env) {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Check existence
const auto binary_path = dir / "kernel.o";
Expand Down Expand Up @@ -159,7 +159,7 @@ class Kernel {
template <typename... Args>
void launch(const LaunchOptions& launch_options, const Args&... args) const {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Checks
DJ_HOST_ASSERT(kernel_handle != nullptr, "kernel must be loaded before launch");
Expand Down
2 changes: 1 addition & 1 deletion include/deep_jit/backend/cuda/backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ class CUDA {
const Config& config,
const CompilerOptions& options) const {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Paths
dir = std::filesystem::absolute(dir).lexically_normal();
Expand Down
8 changes: 4 additions & 4 deletions include/deep_jit/backend/cuda/kernel.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@
#include <memory>
#include <type_traits>

#include <ATen/cuda/CUDAContext.h>
#include <cuda.h>

#include <deep_jit/backend/cuda/driver.hpp>
#include <deep_jit/backend/cuda/options.hpp>
#include <deep_jit/backend/cuda/stable_torch_utils.h>
#include <deep_jit/utils/env.hpp>
#include <deep_jit/utils/exception.hpp>
#include <deep_jit/utils/gil.hpp>
Expand Down Expand Up @@ -47,7 +47,7 @@ class Kernel {

static std::shared_ptr<Kernel> load(const std::filesystem::path& dir, const Env& env) {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Check existence
const auto cubin_path = dir / "kernel.cubin";
Expand Down Expand Up @@ -91,7 +91,7 @@ class Kernel {
template <typename... Args>
void launch(const LaunchOptions& launch_options, const Args&... args) const {
// Release GIL to let other Python threads run
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;

// Checks
DJ_HOST_ASSERT(kernel_handle != nullptr, "kernel must be loaded before launch");
Expand Down Expand Up @@ -163,7 +163,7 @@ class Kernel {
config.sharedMemBytes = *launch_options.num_smem_bytes;
config.hStream = launch_options.stream
? *launch_options.stream
: at::cuda::getCurrentCUDAStream().stream();
: get_current_cuda_stream(torch::stable::accelerator::getCurrentDeviceIndex());
config.attrs = num_attributes == 0 ? nullptr : attributes.data();
config.numAttrs = num_attributes;
DJ_CUDA_DRIVER_CHECK(driver::lazy_cuLaunchKernelEx(
Expand Down
70 changes: 70 additions & 0 deletions include/deep_jit/backend/cuda/stable_torch_utils.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#pragma once

#include <cstdint>

#include <cuda.h>
#include <cuda_runtime_api.h>

// Expose the CUDA declarations in Torch's shims without adding USE_CUDA to
// the rest of the including translation unit.
#ifndef USE_CUDA
#define DEEP_JIT_UNDEFINE_USE_CUDA
#define USE_CUDA
#endif
#include <torch/csrc/stable/accelerator.h>
#include <torch/csrc/stable/c/shim.h>
#include <torch/headeronly/util/shim_utils.h>
#ifdef DEEP_JIT_UNDEFINE_USE_CUDA
#undef USE_CUDA
#undef DEEP_JIT_UNDEFINE_USE_CUDA
#endif

namespace deep_jit::cuda {

// Returns the current CUDA stream for use with the CUDA Driver API.
inline CUstream get_current_cuda_stream(const int32_t device_index) {
void* stream_ptr = nullptr;
TORCH_ERROR_CODE_CHECK(
aoti_torch_get_current_cuda_stream(device_index, &stream_ptr));
return static_cast<CUstream>(stream_ptr);
}

// Upstream torch plans to add stable API's for get_stream_from_pool and TorchCUDAStreamGuard.
// Once they are added and TORCH_TARGET_VERSION catches up, this file can be deleted
// and the APIs can be called directly.

inline cudaStream_t get_stream_from_pool(const int32_t device_index) {
void* stream = nullptr;
TORCH_ERROR_CODE_CHECK(
torch_get_cuda_stream_from_pool(false, device_index, &stream));
return static_cast<cudaStream_t>(stream);
}

class TorchCUDAStreamGuard {
public:
TorchCUDAStreamGuard() = delete;

explicit TorchCUDAStreamGuard(cudaStream_t stream, int32_t device_index) {
TORCH_ERROR_CODE_CHECK(
aoti_torch_create_cuda_stream_guard(
static_cast<void*>(stream), device_index, &guard_));
}

~TorchCUDAStreamGuard() {
if (guard_)
(void)aoti_torch_delete_cuda_stream_guard(guard_);
}

// Match c10::cuda::CUDAStreamGuard: copying is disallowed because the
// guard has unique ownership, and moving is disallowed because an RAII
// stream guard has no uninitialized state to leave behind.
TorchCUDAStreamGuard(const TorchCUDAStreamGuard&) = delete;
TorchCUDAStreamGuard& operator=(const TorchCUDAStreamGuard&) = delete;
TorchCUDAStreamGuard(TorchCUDAStreamGuard&&) = delete;
TorchCUDAStreamGuard& operator=(TorchCUDAStreamGuard&&) = delete;

private:
CUDAStreamGuardHandle guard_ = nullptr;
};

} // namespace deep_jit::cuda
2 changes: 1 addition & 1 deletion include/deep_jit/runtime/runtime.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ class Runtime {
backend.compile(source, entry.path, env, config, options);

// Publish
GilScopedRelease gil_release;
[[maybe_unused]] GilScopedRelease gil_release;
return entry.commit();
}

Expand Down
24 changes: 23 additions & 1 deletion include/deep_jit/utils/gil.hpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
#pragma once

#include <optional>
#ifndef DJ_DISABLE_GIL
#define DJ_DISABLE_GIL 0
#endif

#if !DJ_DISABLE_GIL
#include <optional>
#include <pybind11/pybind11.h>

namespace deep_jit {
Expand All @@ -22,3 +26,21 @@ class GilScopedRelease {
};

} // namespace deep_jit

#else

namespace deep_jit {

class GilScopedRelease {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not very familiar with the nogil support of pytorch upstream, would you mind elaborate a little bit? Does this mean that DeepJIT itself supports nogil? Or downstream libraries (e.g. deepgemm / deepep) needs extra changes.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When kernels are registered with TORCH_LIBRARY, the operator is registered through PyTorch's dispatcher mechanism (basically becomes just like any other PyTorch op) and can be invoked from Python through torch.ops.<op_name>. When a PyTorch operator is invoked from Python, PyTorch releases the GIL before executing the operator through the dispatcher, so by the time it reaches GilScopedRelease gil_release here, the GIL has already been released and that guard is effectively a no-op. This means that DeepJIT doesn't need to explicitly release the GIL for downstream operators that are registered through TORCH_LIBRARY/STABLE_TORCH_LIBRARY.

Currently, DeepGEMM dispatches through pybind, so the explicit GIL release is still required there. We are working on moving DeepGEMM's registration from pybind to TORCH_LIBRARY (see vllm-project/DeepGEMM#15 and deepseek-ai/DeepGEMM#393). We need that migration to happen before we can remove the explicit GIL release from this path, which is why we added a macro flag DJ_DISABLE_GIL rather than removing the release outright. This allows existing downstream users that still dispatch through pybind to continue working correctly while we migrate.

I'm not as familiar with DeepEP, so if it still relies on pybind or otherwise requires the GIL to be released at this point, that flag would need to remain for it as well, or any other downstream task that aren't registered through TORCH_LIBRARY.

Once the downstream libraries have all moved from pybind to TORCH_LIBRARY/STABLE_TORCH_LIBRARY, and assuming they don't otherwise interact with Python while executing their kernels, we should be able to remove the explicit GilScopedRelease entirely.

Let me know if you have any other questions or need anything clarified. Thanks!

public:
GilScopedRelease() = default;

GilScopedRelease(const GilScopedRelease&) = delete;
GilScopedRelease& operator=(const GilScopedRelease&) = delete;
GilScopedRelease(GilScopedRelease&&) = delete;
GilScopedRelease& operator=(GilScopedRelease&&) = delete;
};

} // namespace deep_jit

#endif
2 changes: 2 additions & 0 deletions tests/test_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ def validate_header_self_containment(temporary_dir):
source_path.write_text(f'#include <{header.as_posix()}>\n', encoding='utf-8')
command = [
os.environ.get('CXX', 'c++'), '-std=c++20', '-fsyntax-only', '-Werror',
'-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA',
'-Wno-attributes', '-Wno-deprecated-declarations',
'-Wno-missing-field-initializers', '-Wno-psabi',
str(source_path),
Expand Down Expand Up @@ -787,6 +788,7 @@ def run_worker():
sources=[str(TEST_CUDA_PROJECT / 'main.cpp')],
extra_cflags=[
'-std=c++20', '-O3', '-fPIC', '-Wall', '-Wextra', '-Werror',
'-DTORCH_TARGET_VERSION=0x020a000000000000', '-DUSE_CUDA',
'-Wno-attributes', '-Wno-missing-field-initializers',
'-Wno-psabi', '-Wno-deprecated-declarations',
],
Expand Down
18 changes: 10 additions & 8 deletions tests/test_cuda_proj/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@
#include <vector>

#include <cuda_runtime.h>
#include <c10/cuda/CUDAGuard.h>
#include <pybind11/pybind11.h>
#include <unistd.h>

#include <deep_jit/backend/cuda/backend.hpp>
#include <deep_jit/backend/cuda/stable_torch_utils.h>
#include <deep_jit/cache/memory.hpp>
#include <deep_jit/python_api.hpp>
#include <deep_jit/utils/command.hpp>
Expand All @@ -45,6 +45,8 @@ namespace {
using Runtime = deep_jit::Runtime<deep_jit::CUDA>;
using CompilerOptions = deep_jit::cuda::CompilerOptions;
using LaunchOptions = deep_jit::cuda::LaunchOptions;
using TorchCUDAStreamGuard = deep_jit::cuda::TorchCUDAStreamGuard;
using deep_jit::cuda::get_stream_from_pool;
namespace fs = std::filesystem;

deep_jit::LazyInit<Runtime> python_api_jit(nullptr);
Expand Down Expand Up @@ -1819,34 +1821,34 @@ void test_runtime_launch_features(Runtime& runtime) {

runtime.default_launch_options.stream = std::nullopt;
runtime.default_launch_options.enable_pdl = false;
const auto current_stream = c10::cuda::getStreamFromPool();
const auto current_stream = get_stream_from_pool(device_index);

cudaGraph_t graph = nullptr;
cudaGraphExec_t graph_exec = nullptr;
{
const c10::cuda::CUDAStreamGuard stream_guard(current_stream);
DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream.stream(), cudaStreamCaptureModeGlobal));
const TorchCUDAStreamGuard stream_guard(current_stream, device_index);
DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream, cudaStreamCaptureModeGlobal));
runtime.launch(
kernel, {
.num_smem_bytes = sizeof(int),
.grid_dim = dim3(2, 1, 1),
.block_dim = dim3(32, 1, 1),
},
output, argument_storage, 10);
DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream.stream(), &graph));
DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream, &graph));
}
check_graph(graph);
DJ_CUDA_RUNTIME_CHECK(cudaGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0));
DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream.stream()));
DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream.stream()));
DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream));
DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream));
check_output(43, 44);
DJ_CUDA_RUNTIME_CHECK(cudaGraphExecDestroy(graph_exec));
DJ_CUDA_RUNTIME_CHECK(cudaGraphDestroy(graph));

graph = nullptr;
graph_exec = nullptr;
{
const c10::cuda::CUDAStreamGuard stream_guard(current_stream);
const TorchCUDAStreamGuard stream_guard(current_stream, device_index);
DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal));
runtime.launch(
kernel, {
Expand Down