Repository navigation
Migrate DeepJIT CUDA integration to the PyTorch stable ABI #11
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 3 commits
e90669c
a40ccf5
e090381
8662604
c2e8a2e
e4f3e5b
81ca29e
bad7a43
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,8 +22,9 @@ | |
| #include <vector> | ||
|
|
||
| #include <cuda_runtime.h> | ||
| #include <c10/cuda/CUDAGuard.h> | ||
| #include <pybind11/pybind11.h> | ||
| #include <torch/csrc/stable/c/shim.h> | ||
| #include <torch/headeronly/util/shim_utils.h> | ||
| #include <unistd.h> | ||
|
|
||
| #include <deep_jit/backend/cuda/backend.hpp> | ||
|
|
@@ -52,6 +53,40 @@ int python_api_num_initializations = 0; | |
| std::shared_ptr<Runtime> process_test_runtime; | ||
| std::shared_ptr<Runtime> gil_test_runtime; | ||
|
|
||
| cudaStream_t get_stream_from_pool(const int32_t device_index) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If i'm not mistaken, upstream torch is planning to add stable api for these two, so maybe we could move them to a separate util file for future removal?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, we can definitely move this and similar torch functions to a util file!
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done! |
||
| void* stream = nullptr; | ||
| TORCH_ERROR_CODE_CHECK( | ||
| torch_get_cuda_stream_from_pool(false, device_index, &stream)); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would be good to upstream a high level C++ wrapper API for this one |
||
| return static_cast<cudaStream_t>(stream); | ||
| } | ||
|
|
||
| class TorchCUDAStreamGuard { | ||
| public: | ||
| TorchCUDAStreamGuard() = delete; | ||
|
|
||
| explicit TorchCUDAStreamGuard(cudaStream_t stream, int32_t device_index) { | ||
| TORCH_ERROR_CODE_CHECK( | ||
| aoti_torch_create_cuda_stream_guard( | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This one too |
||
| static_cast<void*>(stream), device_index, &guard_)); | ||
| } | ||
|
|
||
| ~TorchCUDAStreamGuard() { | ||
| if (guard_) | ||
| (void)aoti_torch_delete_cuda_stream_guard(guard_); | ||
| } | ||
|
|
||
| // Match c10::cuda::CUDAStreamGuard: copying is disallowed because the | ||
| // guard has unique ownership, and moving is disallowed because an RAII | ||
| // stream guard has no uninitialized state to leave behind. | ||
| TorchCUDAStreamGuard(const TorchCUDAStreamGuard&) = delete; | ||
| TorchCUDAStreamGuard& operator=(const TorchCUDAStreamGuard&) = delete; | ||
| TorchCUDAStreamGuard(TorchCUDAStreamGuard&&) = delete; | ||
| TorchCUDAStreamGuard& operator=(TorchCUDAStreamGuard&&) = delete; | ||
|
|
||
| private: | ||
| CUDAStreamGuardHandle guard_ = nullptr; | ||
| }; | ||
|
|
||
| const fs::path& get_test_cuda_project_dir() { | ||
| static const fs::path path = [] { | ||
| const auto value = std::getenv("DEEP_JIT_CUDA_TEST_SOURCE_DIR"); | ||
|
|
@@ -1819,34 +1854,34 @@ void test_runtime_launch_features(Runtime& runtime) { | |
|
|
||
| runtime.default_launch_options.stream = std::nullopt; | ||
| runtime.default_launch_options.enable_pdl = false; | ||
| const auto current_stream = c10::cuda::getStreamFromPool(); | ||
| const auto current_stream = get_stream_from_pool(device_index); | ||
|
|
||
| cudaGraph_t graph = nullptr; | ||
| cudaGraphExec_t graph_exec = nullptr; | ||
| { | ||
| const c10::cuda::CUDAStreamGuard stream_guard(current_stream); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream.stream(), cudaStreamCaptureModeGlobal)); | ||
| const TorchCUDAStreamGuard stream_guard(current_stream, device_index); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(current_stream, cudaStreamCaptureModeGlobal)); | ||
| runtime.launch( | ||
| kernel, { | ||
| .num_smem_bytes = sizeof(int), | ||
| .grid_dim = dim3(2, 1, 1), | ||
| .block_dim = dim3(32, 1, 1), | ||
| }, | ||
| output, argument_storage, 10); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream.stream(), &graph)); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaStreamEndCapture(current_stream, &graph)); | ||
| } | ||
| check_graph(graph); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0)); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream.stream())); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream.stream())); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaGraphLaunch(graph_exec, current_stream)); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(current_stream)); | ||
| check_output(43, 44); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaGraphExecDestroy(graph_exec)); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaGraphDestroy(graph)); | ||
|
|
||
| graph = nullptr; | ||
| graph_exec = nullptr; | ||
| { | ||
| const c10::cuda::CUDAStreamGuard stream_guard(current_stream); | ||
| const TorchCUDAStreamGuard stream_guard(current_stream, device_index); | ||
| DJ_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal)); | ||
| runtime.launch( | ||
| kernel, { | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.