From 9fd31ac8e342c8239c4dafa1979aa0bec9f676f7 Mon Sep 17 00:00:00 2001 From: Songhao Jia Date: Mon, 7 Apr 2025 12:25:57 -0700 Subject: [PATCH] introducing filter to etdumpgen (#9937) Summary: This diff introduce etdump filter to etdumpgen. Detail design can be found in Details can be found in https://github.com/pytorch/executorch/discussions/9260 Reviewed By: tarun292 Differential Revision: D72151631 --- devtools/etdump/etdump_filter.h | 5 +- devtools/etdump/etdump_flatcc.cpp | 18 ++++ devtools/etdump/etdump_flatcc.h | 10 ++ devtools/etdump/tests/etdump_test.cpp | 129 +++++++++++++++++++++++- devtools/etdump/tests/targets.bzl | 1 + runtime/core/event_tracer.h | 12 +-- runtime/core/test/event_tracer_test.cpp | 6 ++ 7 files changed, 172 insertions(+), 9 deletions(-) diff --git a/devtools/etdump/etdump_filter.h b/devtools/etdump/etdump_filter.h index 545823a5556..29db43be8b9 100644 --- a/devtools/etdump/etdump_filter.h +++ b/devtools/etdump/etdump_filter.h @@ -77,8 +77,9 @@ class ETDumpFilter : public ::executorch::runtime::EventTracerFilterBase { * * @return A Result indicating whether the event matches the filter * criteria. - * - True if the event matches the filter, or filter is unset. - * - False if the event does not match or is unknown. + * - True if the event matches the filter. + * - False if the event does not match, or is unknown, or filter is + * unset. * - An error code if an error occurs during filtering. */ Result filter( diff --git a/devtools/etdump/etdump_flatcc.cpp b/devtools/etdump/etdump_flatcc.cpp index ea464f2f5ce..f6e18bf49d2 100644 --- a/devtools/etdump/etdump_flatcc.cpp +++ b/devtools/etdump/etdump_flatcc.cpp @@ -369,6 +369,19 @@ Result ETDumpGen::log_intermediate_output_delegate_helper( InvalidArgument, "Only name or delegate_debug_index can be valid. Check DelegateMappingBuilder documentation for more details."); + if (filter_) { + Result result = filter_->filter(name, delegate_debug_index); + if (!result.ok()) { + return result; + } + + // If the filter returns true, meaning this event should be filtered out and + // we should not log it. + if (result.get()) { + return false; + } + } + check_ready_to_add_events(); int64_t string_id = name != nullptr ? create_string_entry(name) : -1; @@ -654,6 +667,11 @@ DataSinkBase* ETDumpGen::get_data_sink() { return data_sink_; } +void ETDumpGen::set_delegation_intermediate_output_filter( + EventTracerFilterBase* filter) { + filter_ = filter; +} + long ETDumpGen::write_tensor_or_raise_error(Tensor tensor) { // Previously, the function copy_tensor_to_debug_buffer returned 0xFF..F when // given an empty tensor, which is an invalid offset for most buffers. In our diff --git a/devtools/etdump/etdump_flatcc.h b/devtools/etdump/etdump_flatcc.h index 6b51745eee3..dc0f822f922 100644 --- a/devtools/etdump/etdump_flatcc.h +++ b/devtools/etdump/etdump_flatcc.h @@ -25,6 +25,7 @@ namespace executorch { namespace etdump { using ::executorch::runtime::DelegateDebugIntId; +using ::executorch::runtime::EventTracerFilterBase; using ::executorch::runtime::Result; namespace internal { @@ -146,6 +147,13 @@ class ETDumpGen : public ::executorch::runtime::EventTracer { const char* name, DelegateDebugIntId delegate_debug_index, const double& output) override; + + /** + * Set the filter of event tracer for delegation intermediate outputs. + */ + virtual void set_delegation_intermediate_output_filter( + EventTracerFilterBase* event_tracer_filter) override; + void set_debug_buffer(::executorch::runtime::Span buffer); void set_data_sink(DataSinkBase* data_sink); ETDumpResult get_etdump_data(); @@ -188,6 +196,8 @@ class ETDumpGen : public ::executorch::runtime::EventTracer { int bundled_input_index_ = -1; State state_ = State::Init; struct internal::ETDumpStaticAllocator alloc_; + + EventTracerFilterBase* filter_ = nullptr; }; } // namespace etdump diff --git a/devtools/etdump/tests/etdump_test.cpp b/devtools/etdump/tests/etdump_test.cpp index c64bab0448c..9d39a8bbde1 100644 --- a/devtools/etdump/tests/etdump_test.cpp +++ b/devtools/etdump/tests/etdump_test.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -33,6 +34,7 @@ using ::executorch::runtime::AllocatorID; using ::executorch::runtime::ArrayRef; using ::executorch::runtime::BoxedEvalueList; using ::executorch::runtime::DelegateDebugIdType; +using ::executorch::runtime::DelegateDebugIntId; using ::executorch::runtime::Error; using ::executorch::runtime::EValue; using ::executorch::runtime::EventTracerEntry; @@ -45,6 +47,8 @@ using ::executorch::runtime::testing::TensorFactory; using ::executorch::etdump::BufferDataSink; using ::executorch::etdump::FileDataSink; +using ::executorch::etdump::ETDumpFilter; + class ProfilerETDumpTest : public ::testing::Test { protected: void SetUp() override { @@ -75,6 +79,70 @@ class ProfilerETDumpTest : public ::testing::Test { "Must set data sink before writing tensor-like data"); } + void check_log_with_filter( + const char* name, + DelegateDebugIntId delegate_debug_index, + bool use_tensor_input, + bool expected_log, + bool expected_ok) { + TensorFactory tf; + for (size_t i = 0; i < 2; i++) { + const size_t buffer_size = 2048; + + void* ptr = malloc(buffer_size); + auto buffer_data_sink = BufferDataSink::create(ptr, buffer_size); + auto filter = ETDumpFilter(); + filter.add_regex("filtered.*"); + filter.set_debug_handle_range(1, 10); + etdump_gen[i]->set_delegation_intermediate_output_filter(&filter); + + etdump_gen[i]->create_event_block("test_block"); + etdump_gen[i]->set_data_sink(&buffer_data_sink.get()); + + // size of empty etdump + size_t initial_size = 68; + + // Perform logging + + if (use_tensor_input) { + auto tensor = tf.ones({3, 2}); + auto result = etdump_gen[i]->log_intermediate_output_delegate( + name, delegate_debug_index, tensor); + ASSERT_EQ(result.ok(), expected_ok); + if (expected_ok) { + ASSERT_EQ(result.get(), expected_log); + } + } else { // use tensor_list instead + std::vector tensors = {tf.ones({5, 4}), tf.ones({7, 6})}; + Result result = etdump_gen[i]->log_intermediate_output_delegate( + name, + delegate_debug_index, + ArrayRef(tensors.data(), tensors.size())); + ASSERT_EQ(result.ok(), expected_ok); + if (expected_ok) { + ASSERT_EQ(result.get(), expected_log); + } + } + + // Get final size of etdump + ETDumpResult final_result = etdump_gen[i]->get_etdump_data(); + size_t final_size = final_result.size; + // Check if the size of etdump has changed based on logging success + if (expected_log) { + ASSERT_NE(initial_size, final_size); // Expect size change if logged + } else { + ASSERT_EQ( + initial_size, final_size); // Expect no size change if not logged + } + + if (!etdump_gen[i]->is_static_etdump()) { + free(final_result.buf); + } + + free(ptr); + } + } + ETDumpGen* etdump_gen[2]; uint8_t* buf = nullptr; std::unique_ptr temp_file; @@ -652,7 +720,7 @@ TEST_F(ProfilerETDumpTest, VerifyDelegateIntermediateLogging) { void* ptr = malloc(2048); Span buffer((uint8_t*)ptr, 2048); - ; + auto buffer_data_sink = BufferDataSink::create(ptr, 2048); auto file_data_sink = FileDataSink::create(dump_file_path.c_str()); @@ -892,3 +960,62 @@ TEST_F(ProfilerETDumpTest, WriteAfterGetETDumpData) { } } } + +TEST_F(ProfilerETDumpTest, LogWithRegexAndUnsetDelegateDebugIdOnTensor) { + check_log_with_filter( + "filtered_event", + kUnsetDelegateDebugIntId, + /*use_tensor_input=*/true, + /*expected_log=*/false, + /*expected_ok=*/true); +} + +TEST_F(ProfilerETDumpTest, LogWithRegexAndUnsetDelegateDebugIdOnTensorList) { + check_log_with_filter( + "filtered_event", + kUnsetDelegateDebugIntId, + /*use_tensor_input=*/true, + /*expected_log=*/false, + /*expected_ok=*/true); +} + +TEST_F(ProfilerETDumpTest, LogWithNullptrAndInRange) { + check_log_with_filter( + nullptr, + 5, + /*use_tensor_input=*/true, + /*expected_log=*/false, + /*expected_ok=*/true); +} +TEST_F(ProfilerETDumpTest, LogWithNonMatchingRegexAndOutOfRange) { + check_log_with_filter( + "unfiltered_event", + kUnsetDelegateDebugIntId, + /*use_tensor_input=*/true, + /*expected_log=*/true, + /*expected_ok=*/true); +} +TEST_F(ProfilerETDumpTest, LogWithNullptrAndOutOfRange) { + check_log_with_filter( + nullptr, + 20, + /*use_tensor_input=*/true, + /*expected_log=*/true, + /*expected_ok=*/true); +} +TEST_F(ProfilerETDumpTest, LogWithRegexAndInRange) { + check_log_with_filter( + "filtered_event", + 5, + /*use_tensor_input=*/true, + /*expected_log=*/false, + /*expected_ok=*/false); +} +TEST_F(ProfilerETDumpTest, LogWithNullptrAndUnsetDebugHandle) { + check_log_with_filter( + nullptr, + kUnsetDelegateDebugIntId, + /*use_tensor_input=*/true, + /*expected_log=*/false, + /*expected_ok=*/false); +} diff --git a/devtools/etdump/tests/targets.bzl b/devtools/etdump/tests/targets.bzl index 7f266eed5a7..10eb8608362 100644 --- a/devtools/etdump/tests/targets.bzl +++ b/devtools/etdump/tests/targets.bzl @@ -19,6 +19,7 @@ def define_common_targets(): "//executorch/extension/testing_util:temp_file", "//executorch/runtime/platform:platform", "//executorch/runtime/core/exec_aten/testing_util:tensor_util", + "//executorch/devtools/etdump:etdump_filter", ], ) diff --git a/runtime/core/event_tracer.h b/runtime/core/event_tracer.h index 77d7fc64102..e810437f7d2 100644 --- a/runtime/core/event_tracer.h +++ b/runtime/core/event_tracer.h @@ -439,6 +439,12 @@ class EventTracer { DelegateDebugIntId delegate_debug_index, const double& output) = 0; + /** + * Set the filter of event tracer for delegation intermediate outputs. + */ + virtual void set_delegation_intermediate_output_filter( + EventTracerFilterBase* event_tracer_filter) = 0; + /** * Helper function to set the chain id ands debug handle. Users have two * options, the first is that they can directly pass in the chain id and debug @@ -513,12 +519,6 @@ class EventTracer { event_tracer_profiling_level_ = profiling_level; } - /** - * Set the filter of event tracer for delegation intermediate outputs. - */ - void set_delegation_intermediate_output_filter( - EventTracerFilterBase* event_tracer_filter); - /** * Return the current level of event tracer profiling. */ diff --git a/runtime/core/test/event_tracer_test.cpp b/runtime/core/test/event_tracer_test.cpp index 224e87cc2b1..ee16aa924a6 100644 --- a/runtime/core/test/event_tracer_test.cpp +++ b/runtime/core/test/event_tracer_test.cpp @@ -28,6 +28,7 @@ using executorch::runtime::EValue; using executorch::runtime::EventTracer; using executorch::runtime::EventTracerDebugLogLevel; using executorch::runtime::EventTracerEntry; +using executorch::runtime::EventTracerFilterBase; using executorch::runtime::kUnsetChainId; using executorch::runtime::kUnsetDebugHandle; using executorch::runtime::kUnsetDelegateDebugIntId; @@ -90,6 +91,11 @@ class DummyEventTracer : public EventTracer { (void)metadata_len; } + void set_delegation_intermediate_output_filter( + EventTracerFilterBase* event_tracer_filter) override { + (void)event_tracer_filter; + } + void log_profiling_delegate( const char* name, DelegateDebugIntId delegate_debug_id,