Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions hta/analyzers/critical_path_analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -1103,6 +1103,18 @@ def _check_event_sync_helper(self, row) -> bool:
)
return False

# Older Kineto builds can name a later record of the same CUDA event;
# a wait cannot depend on a record issued after it.
if row.wait_on_cuda_event_record_corr_id > row.correlation:
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"CUDA Stream Wait event names a cudaEventRecord issued after "
f"it, name = {self._get_node_name(eid)}, correlation = "
f"{row.correlation}, record correlation = "
f"{row.wait_on_cuda_event_record_corr_id}"
)
return False

if (not hasattr(row, "index_previous_launch")) or (
row.index_previous_launch == -1
):
Expand Down Expand Up @@ -1260,6 +1272,11 @@ def handle_cuda_sync(row):
]

if name == stream_wait_event:
# Stream order already covers a wait on an event recorded
# on the same stream.
if row.wait_on_stream == row.stream:
return

# Stream Wait event is indicating a dependency between
# the next GPU kernel on this stream and another GPU kernel

Expand Down
99 changes: 98 additions & 1 deletion tests/test_critical_path_analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from collections import Counter
from dataclasses import dataclass, field
from tempfile import TemporaryDirectory
from typing import Dict, List, Tuple
from typing import Any, Callable, Dict, List, Tuple
from unittest.mock import patch

import pandas as pd
Expand Down Expand Up @@ -985,6 +985,103 @@ def test_mtia_trace(self) -> None:
f"launch-delay sources={sorted(launch_delay_runtime_indices)}",
)

def _edited_event_sync_trace(
self, trace_dir: str, edit: Callable[[List[Dict[str, Any]]], None]
) -> TraceAnalysis:
"""The multi-stream event sync trace after `edit` changes its events."""
src = os.path.join(
get_test_data_dir("critical_path/cuda_event_sync_multi_stream"),
"event_sync_trace.json.gz",
)
with gzip.open(src, "rt") as f:
trace = json.load(f)
edit(trace["traceEvents"])
with gzip.open(os.path.join(trace_dir, "event_sync_trace.json.gz"), "wt") as f:
json.dump(trace, f)
return TraceAnalysis(trace_dir=trace_dir)

@staticmethod
def _sync_event(events: List[Dict[str, Any]], correlation: int) -> Dict[str, Any]:
return next(
e
for e in events
if e.get("cat") == "cuda_sync" and e["args"]["correlation"] == correlation
)

def _assert_critical_path_succeeds(
self, edit: Callable[[List[Dict[str, Any]]], None]
) -> None:
with TemporaryDirectory() as trace_dir:
trace = self._edited_event_sync_trace(trace_dir, edit)
_, success = trace.critical_path_analysis(
rank=0, annotation="", instance_id=None
)
self.assertTrue(success)

def _wait_on_later_record(
self, events: List[Dict[str, Any]], wait_correlation: int
) -> None:
"""Points the wait at cudaEventRecord 1418, issued after it on stream
24, as older Kineto builds do when the event is recorded again."""
args = self._sync_event(events, wait_correlation)["args"]
args["wait_on_cuda_event_record_corr_id"] = 1418
args["wait_on_stream"] = 24

def test_critical_path_ignores_event_sync_on_later_record(self) -> None:
"""cudaEventQuery (correlation 54) cannot wait on a record issued after
it."""
self._assert_critical_path_succeeds(
lambda events: self._wait_on_later_record(events, 54)
)

def test_critical_path_ignores_stream_wait_on_later_record(self) -> None:
"""cudaStreamWaitEvent on stream 24 (correlation 1389) cannot wait on a
record issued after it, here on stream 24 itself."""
self._assert_critical_path_succeeds(
lambda events: self._wait_on_later_record(events, 1389)
)

def test_critical_path_ignores_stream_wait_on_same_stream(self) -> None:
"""A wait on stream 20 for cudaEventRecord 1385, also recorded on
stream 20, adds nothing to stream order before the next kernel there."""

def edit(events: List[Dict[str, Any]]) -> None:
wait = self._sync_event(events, 1389)
wait["tid"] = wait["args"]["stream"] = 20
launch_ts = wait["ts"] + 10
events.append(
{
"ph": "X",
"cat": "cuda_runtime",
"name": "cudaLaunchKernel",
"pid": 3727853,
"tid": 3727853,
"ts": launch_ts,
"dur": 10,
"args": {"External id": 7, "cbid": 211, "correlation": 1390},
}
)
events.append(
{
"ph": "X",
"cat": "kernel",
"name": "ampere_sgemm_128x64_nn",
"pid": 0,
"tid": 20,
"ts": launch_ts + 20,
"dur": 50,
"args": {
"External id": 7,
"device": 0,
"context": 1,
"stream": 20,
"correlation": 1390,
},
}
)

self._assert_critical_path_succeeds(edit)


class EndToEndTestCase(unittest.TestCase):
"""Tests the input / final output (overlaid trace file) of critical path analysis.
Expand Down
Loading