Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions tests/models/deepseek/test_deepseek_qwen.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ def _load_inputs(self):
def test_deepseek_qwen(record_property, model_name, mode, op_by_op):
if mode == "train":
pytest.skip()
pytest.skip("#291")

cc = CompilerConfig()
if op_by_op:
Expand All @@ -54,7 +53,11 @@ def test_deepseek_qwen(record_property, model_name, mode, op_by_op):
cc.op_by_op_backend = OpByOpBackend.STABLEHLO

tester = ThisTester(
model_name, mode, compiler_config=cc, record_property_handle=record_property
model_name,
mode,
compiler_config=cc,
record_property_handle=record_property,
required_atol=0.5,
)

tester.test_model()
Expand Down
112 changes: 109 additions & 3 deletions tt_torch/dynamo/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@
import onnxruntime as ort
import tt_mlir
import time
import pickle
import faulthandler
import sys
import re
import tempfile
import os
import multiprocessing as mp
from tt_torch.tools.utils import (
CompileDepth,
Expand All @@ -21,6 +23,34 @@
from typing import Union


def gb_to_bytes(gb):
return gb * 1024 * 1024 * 1024


def get_tensor_size(tensor):
"""Calculate the memory size of a tensor in bytes."""
Comment thread
ddilbazTT marked this conversation as resolved.
Resolved
if isinstance(tensor, torch.Tensor):
return tensor.element_size() * tensor.nelement()
return 0


def get_inputs_size(inputs):
"""Calculate the total memory size of inputs in bytes."""
total_size = 0

if isinstance(inputs, torch.Tensor):
total_size += get_tensor_size(inputs)
elif isinstance(inputs, (list, tuple)):
for item in inputs:
total_size += get_inputs_size(item)
elif isinstance(inputs, dict):
for item in inputs.values():
total_size += get_inputs_size(item)
else:
assert false, f"Unexpected input type: {type(inputs)}"
return total_size


def compile_process(receiver, sender, ttir_event, ttnn_event, json_event):
obj = receiver.get()
faulthandler.disable()
Expand All @@ -36,24 +66,61 @@ def compile_process(receiver, sender, ttir_event, ttnn_event, json_event):
sys.exit(0)


def print_tensor_shapes(inputs, prefix=""):
"""Print the shapes and dtypes of tensor inputs for debugging."""
if isinstance(inputs, torch.Tensor):
print(f"{prefix}dtype = {inputs.dtype}, shape = {inputs.shape}")
elif isinstance(inputs, (list, tuple)):
print(f"{prefix}List/Tuple of length {len(inputs)}")
for i, item in enumerate(inputs):
print_tensor_shapes(item, prefix=f"{prefix}[{i}].")
elif isinstance(inputs, dict):
print(f"{prefix}Dict with keys: {list(inputs.keys())}")
for key, item in inputs.items():
print_tensor_shapes(item, prefix=f"{prefix}['{key}'].")
else:
print(f"{prefix}Not a tensor: {type(inputs)}")


def execute_process(receiver, sender, exec_event):
while 1:
obj = receiver.get()
faulthandler.disable()
binary = obj["binary"]
inputs = obj["inputs"]
file_name = obj["dump_file"]
large_input = obj["large_input"]
inputs = None

# Load inputs from disk if they're large
if large_input:
print("Child process handling large input", flush=True)
inputs_file_path = obj["inputs_file_path"]
if inputs_file_path and os.path.exists(inputs_file_path):
try:
with open(inputs_file_path, "rb") as f:
inputs = pickle.load(f)
except Exception as e:
print(f"Error loading inputs from disk: {e}")
else:
inputs = obj["inputs"]

file_stderr = open(file_name, "w")
old_stderr = sys.stderr
sys.stderr = file_stderr
old_stdout = sys.stdout
sys.stdout = file_stderr
outputs = tt_mlir.run_end_to_end(inputs, binary)

outputs = None
if inputs is not None:
outputs = tt_mlir.run_end_to_end(inputs, binary)

sys.stderr = old_stderr
sys.stdout = old_stdout
file_stderr.close()

sender.put({"outputs": outputs})
exec_event.wait()

sys.exit(0)


Expand Down Expand Up @@ -246,6 +313,7 @@ def __init__(
# to capture runtime stack dump.
self.stderror_redirected = False
self.file_stderr = None
self.op_memory_limit = gb_to_bytes(1) # 1GB limit

def transform_input(self, inp):
# Convert torch.nn.Parameter to torch.Tensor and convert non-contiguous
Expand Down Expand Up @@ -381,7 +449,39 @@ def run_op(self, binary, *inputs):
self.file_stderr = tempfile.NamedTemporaryFile(mode="w+t", delete=False)
self.stderror_redirected = True

obj = {"binary": binary, "inputs": inputs, "dump_file": self.file_stderr.name}
inputs_size = get_inputs_size(inputs)

large_input = inputs_size > self.op_memory_limit

obj = {
"binary": binary,
"dump_file": self.file_stderr.name,
"large_input": large_input,
}

inputs_file_path = None
if large_input:
obj["inputs"] = None
try:
Comment thread
ddilbazTT marked this conversation as resolved.
Resolved
temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".pkl")
Comment thread
ddilbazTT marked this conversation as resolved.
Resolved
inputs_file_path = temp_file.name
temp_file.close()

with open(inputs_file_path, "wb") as f:
pickle.dump(inputs, f)

obj["inputs_file_path"] = inputs_file_path
except Exception as e:
print(f"Error saving inputs to disk: {e}")
if inputs_file_path and os.path.exists(inputs_file_path):
try:
os.remove(inputs_file_path)
except OSError:
pass
large_input = False
else:
obj["inputs"] = inputs
Comment thread
ddilbazTT marked this conversation as resolved.
Resolved
obj["inputs_file_path"] = None

exec_event = mp.Event()
if self.execute_process is None:
Expand Down Expand Up @@ -412,6 +512,12 @@ def run_op(self, binary, *inputs):
self.execute_process = None
break

if os.path.isfile(inputs_file_path):
try:
os.remove(inputs_file_path)
except OSError:
pass

if len(outputs) == 1:
outputs = outputs[0]

Expand Down