diff --git a/extension/flat_tensor/TARGETS b/extension/flat_tensor/TARGETS new file mode 100644 index 00000000000..08e83a5f3c4 --- /dev/null +++ b/extension/flat_tensor/TARGETS @@ -0,0 +1,6 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load(":targets.bzl", "define_common_targets") + +oncall("executorch") + +define_common_targets() diff --git a/extension/flat_tensor/flat_tensor_data_map.cpp b/extension/flat_tensor/flat_tensor_data_map.cpp new file mode 100644 index 00000000000..b6d03f88148 --- /dev/null +++ b/extension/flat_tensor/flat_tensor_data_map.cpp @@ -0,0 +1,257 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +using executorch::runtime::Error; +using executorch::runtime::FreeableBuffer; +using executorch::runtime::Result; +using executorch::runtime::Span; + +using executorch::aten::ScalarType; +using executorch::runtime::DataLoader; +using executorch::runtime::TensorLayout; + +namespace executorch { +namespace extension { + +namespace { +/** + * FlatTensor data must be aligned to this value to properly parse it. Must be a + * power of 2. Note that max_align_t is the alignment that malloc() and new + * guarantee. + */ +constexpr size_t kMinimumAlignment = alignof(std::max_align_t); + +bool is_aligned(const void* data) { + uintptr_t addr = reinterpret_cast(data); + return addr % kMinimumAlignment == 0; +} + +Result get_flat_tensor_metadata( + const char* key, + const flatbuffers::Vector< + flatbuffers::Offset>* tensors) { + // Linear search by name. + for (int i = 0; i < tensors->size(); i++) { + if (std::strcmp(tensors->Get(i)->fully_qualified_name()->c_str(), key) == + 0) { + // TODO(T214294528): Support multiple segments in FlatTensor. + if (tensors->Get(i)->segment_index() != 0) { + return Error::InvalidExternalData; + } + return tensors->Get(i); + } + } + return Error::NotFound; +} + +Result create_tensor_layout( + const flat_tensor_flatbuffer::TensorMetadata* tensor_metadata) { + ScalarType scalar_type = + static_cast(tensor_metadata->scalar_type()); + const int dim = tensor_metadata->sizes()->size(); + const auto serialized_sizes = tensor_metadata->sizes()->data(); + const auto serialized_dim_order = tensor_metadata->dim_order()->data(); + return TensorLayout::create( + Span(serialized_sizes, dim), + Span(serialized_dim_order, dim), + scalar_type); +} + +} // namespace + +ET_NODISCARD Result FlatTensorDataMap::get_metadata( + const char* key) const { + Result metadata_res = + get_flat_tensor_metadata(key, flat_tensor_->tensors()); + if (!metadata_res.ok()) { + return metadata_res.error(); + } + return create_tensor_layout(metadata_res.get()); +} + +ET_NODISCARD Result FlatTensorDataMap::get_data( + const char* key) const { + auto tensor_metadata = flat_tensor_->tensors(); + + Result metadata_res = + get_flat_tensor_metadata(key, tensor_metadata); + if (!metadata_res.ok()) { + return metadata_res.error(); + } + const auto metadata = metadata_res.get(); + if (metadata->segment_index() < 0 || metadata->offset() < 0) { + // Invalid segment_index/offset; malformed PTD file. + return Error::InvalidExternalData; + } + + Result tensor_layout_res = create_tensor_layout(metadata); + if (!tensor_layout_res.ok()) { + return tensor_layout_res.error(); + } + + // This FreeableBuffer doesn't own the underlying data, and will not free it, + // which is why the free function is a nullptr. + // TODO(T214294528): Remove data_ro_ and instead load the data here, letting + // FreeableBuffer own it. + return FreeableBuffer( + static_cast(data_ro_.data()) + metadata->offset(), + tensor_layout_res.get().nbytes(), + nullptr); +} + +ET_NODISCARD Result FlatTensorDataMap::load_data_into( + ET_UNUSED const char* key, + ET_UNUSED void* buffer, + ET_UNUSED size_t size) const { + return Error::NotImplemented; +} + +ET_NODISCARD Result FlatTensorDataMap::get_num_keys() const { + return flat_tensor_->tensors()->size(); +} + +ET_NODISCARD Result FlatTensorDataMap::get_key( + size_t index) const { + if (index < 0 || index >= flat_tensor_->tensors()->size()) { + return Error::InvalidArgument; + } + return flat_tensor_->tensors()->Get(index)->fully_qualified_name()->c_str(); +} + +/* static */ Result FlatTensorDataMap::load( + DataLoader* loader) { + // Load data map. + size_t flatbuffer_offset = 0; + size_t flatbuffer_size = 0; + size_t segment_base_offset = 0; + size_t segment_data_size = 0; + { + // Check header. + Result header = loader->load( + /*offset=*/0, + FlatTensorHeader::kNumHeadBytes, + DataLoader::SegmentInfo(DataLoader::SegmentInfo::Type::External)); + if (!header.ok()) { + return header.error(); + } + Result fh = + FlatTensorHeader::Parse(header->data(), header->size()); + if (fh.ok()) { + // The header has the data map size. + flatbuffer_offset = fh->flatbuffer_offset; + flatbuffer_size = fh->flatbuffer_size; + segment_base_offset = fh->segment_base_offset; + segment_data_size = fh->segment_data_size; + } else if (fh.error() == Error::NotFound) { + // No header, throw error. + ET_LOG(Error, "No FlatTensorHeader found."); + return fh.error(); + } else { + // corruption, throw error. + ET_LOG(Error, "Flat tensor header may be corrupt."); + return fh.error(); + } + } + + // Load flatbuffer data as a segment. + Result flat_tensor_data = loader->load( + /*offset=*/0, + flatbuffer_offset + flatbuffer_size, + DataLoader::SegmentInfo(DataLoader::SegmentInfo::Type::External)); + if (!flat_tensor_data.ok()) { + return flat_tensor_data.error(); + } + + // Make sure magic matches. + if (!flat_tensor_flatbuffer::FlatTensorBufferHasIdentifier( + flat_tensor_data->data())) { + ET_LOG( + Error, + "FlatTensor identifier '%.4s' != expected '%.4s'", + flatbuffers::GetBufferIdentifier(flat_tensor_data->data()), + flat_tensor_flatbuffer::FlatTensorIdentifier()); + return Error::InvalidExternalData; + } + + // The flatbuffer data must start at an aligned address to ensure internal + // alignment of flatbuffer fields. + ET_CHECK_OR_RETURN_ERROR( + is_aligned(flat_tensor_data->data()), + InvalidArgument, + "FlatTensor data 0x%p must be aligned to %zu", + flat_tensor_data->data(), + kMinimumAlignment); + + // Get pointer to root of flatbuffer table. + const flat_tensor_flatbuffer::FlatTensor* flat_tensor = + flat_tensor_flatbuffer::GetFlatTensor(flat_tensor_data->data()); + + // Validate flatbuffer data. + flatbuffers::Verifier verifier( + reinterpret_cast(flat_tensor_data->data()), + flat_tensor_data->size()); + bool ok = flat_tensor_flatbuffer::VerifyFlatTensorBuffer(verifier); + ET_CHECK_OR_RETURN_ERROR( + ok, + InvalidExternalData, + "Verification failed; data may be truncated or corrupt"); + + // Get pointer to tensor metadata. + const auto* s_tensor_metadata = flat_tensor->tensors(); + if (s_tensor_metadata == nullptr) { + ET_LOG(Error, "FlatTensor has no tensor metadata."); + return Error::InvalidExternalData; + } + + // Load constant data. + const auto* s_data_segment = flat_tensor->segments(); + + // TODO(T214294528): Support multiple segments in FlatTensor. + if (s_data_segment->size() != 1) { + ET_LOG( + Error, + "FlatTensor has %u segments, only 1 supported.", + s_data_segment->size()); + } + // First segment size should be <= the total segment data size. + int segment_size = s_data_segment->Get(0)->size(); + int segment_offset = s_data_segment->Get(0)->offset(); + if (segment_size > segment_data_size) { + ET_LOG( + Error, + "FlatTensor segment size %d > segment data size %zu", + segment_size, + segment_data_size); + } + + Result data_ro = loader->load( + /*offset=*/segment_base_offset + segment_offset, + segment_size, + DataLoader::SegmentInfo(DataLoader::SegmentInfo::Type::External)); + if (!data_ro.ok()) { + return data_ro.error(); + } + + return FlatTensorDataMap( + std::move(flat_tensor_data.get()), flat_tensor, std::move(data_ro.get())); +} + +} // namespace extension +} // namespace executorch diff --git a/extension/flat_tensor/flat_tensor_data_map.h b/extension/flat_tensor/flat_tensor_data_map.h new file mode 100644 index 00000000000..7bd33e68927 --- /dev/null +++ b/extension/flat_tensor/flat_tensor_data_map.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include + +// Forward declare flatbuffer types. This is a public header and must not +// include the generated flatbuffer header. +namespace flat_tensor_flatbuffer { +struct FlatTensor; +} // namespace flat_tensor_flatbuffer + +namespace executorch { +namespace extension { + +/** + * A NamedDataMap implementation for FlatTensor-serialized data. + */ +class FlatTensorDataMap final : public executorch::runtime::NamedDataMap { + public: + /** + * Creates a new DataMap that wraps FlatTensor data. + * + * @param[in] loader The DataLoader that wraps the FlatTensor file. + * Note: the loader must outlive the FlatTensorDataMap instance. + */ + static executorch::runtime::Result load( + executorch::runtime::DataLoader* loader); + + ET_NODISCARD + executorch::runtime::Result + get_metadata(const char* key) const override; + ET_NODISCARD + executorch::runtime::Result get_data( + const char* key) const override; + ET_NODISCARD executorch::runtime::Result + load_data_into(const char* key, void* buffer, size_t size) const override; + + ET_NODISCARD executorch::runtime::Result get_num_keys() + const override; + ET_NODISCARD executorch::runtime::Result get_key( + size_t index) const override; + + FlatTensorDataMap(FlatTensorDataMap&&) noexcept = default; + + ~FlatTensorDataMap() override = default; + + private: + FlatTensorDataMap( + executorch::runtime::FreeableBuffer&& flat_tensor_data, + const flat_tensor_flatbuffer::FlatTensor* flat_tensor, + executorch::runtime::FreeableBuffer&& data_ro) + : flat_tensor_data_(std::move(flat_tensor_data)), + flat_tensor_(flat_tensor), + data_ro_(std::move(data_ro)) {} + + // Not copyable or assignable. + FlatTensorDataMap(const FlatTensorDataMap& rhs) = delete; + FlatTensorDataMap& operator=(FlatTensorDataMap&& rhs) noexcept = delete; + FlatTensorDataMap& operator=(const FlatTensorDataMap& rhs) = delete; + + // Serialized flat_tensor flatbuffer data. + executorch::runtime::FreeableBuffer flat_tensor_data_; + + // Flatbuffer representation of the flat_tensor. + const flat_tensor_flatbuffer::FlatTensor* flat_tensor_; + + // Loaded read-only tensor data. + executorch::runtime::FreeableBuffer data_ro_; +}; + +} // namespace extension +} // namespace executorch diff --git a/extension/flat_tensor/targets.bzl b/extension/flat_tensor/targets.bzl new file mode 100644 index 00000000000..ed2adefc581 --- /dev/null +++ b/extension/flat_tensor/targets.bzl @@ -0,0 +1,22 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +def define_common_targets(): + runtime.cxx_library( + name = "flat_tensor_data_map", + srcs = [ + "flat_tensor_data_map.cpp", + ], + exported_headers = ["flat_tensor_data_map.h"], + deps = [ + "//executorch/extension/flat_tensor/serialize:generated_headers", + "//executorch/extension/flat_tensor/serialize:flat_tensor_header", + "//executorch/runtime/core:core", + "//executorch/runtime/core:evalue", + "//executorch/runtime/core:named_data_map", + "//executorch/runtime/core/exec_aten:lib", + "//executorch/runtime/core/exec_aten/util:tensor_util", + ], + visibility = [ + "//executorch/...", + ], + ) diff --git a/extension/flat_tensor/test/TARGETS b/extension/flat_tensor/test/TARGETS index c9989b67554..0dc97974af8 100644 --- a/extension/flat_tensor/test/TARGETS +++ b/extension/flat_tensor/test/TARGETS @@ -6,7 +6,7 @@ load(":targets.bzl", "define_common_targets") oncall("executorch") -define_common_targets() +define_common_targets(is_fbcode=True) python_unittest( name = "serialize", diff --git a/extension/flat_tensor/test/flat_tensor_data_map_test.cpp b/extension/flat_tensor/test/flat_tensor_data_map_test.cpp new file mode 100644 index 00000000000..9673f5c850c --- /dev/null +++ b/extension/flat_tensor/test/flat_tensor_data_map_test.cpp @@ -0,0 +1,139 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace ::testing; +using executorch::extension::FlatTensorDataMap; +using executorch::extension::FlatTensorHeader; +using executorch::runtime::DataLoader; +using executorch::runtime::Error; +using executorch::runtime::FreeableBuffer; +using executorch::runtime::Result; +using executorch::runtime::TensorLayout; +using torch::executor::util::FileDataLoader; + +class FlatTensorDataMapTest : public ::testing::Test { + protected: + void SetUp() override { + // Since these tests cause ET_LOG to be called, the PAL must be initialized + // first. + executorch::runtime::runtime_init(); + + // Load data map. The eager linear model is defined at: + // //executorch/test/models/linear_model.py + const char* path = std::getenv("ET_MODULE_LINEAR_DATA"); + Result loader = FileDataLoader::from(path); + ASSERT_EQ(loader.error(), Error::Ok); + + data_map_loader_ = + std::make_unique(std::move(loader.get())); + } + std::unique_ptr data_map_loader_; +}; + +TEST_F(FlatTensorDataMapTest, LoadFlatTensorDataMap) { + Result data_map = + FlatTensorDataMap::load(data_map_loader_.get()); + EXPECT_EQ(data_map.error(), Error::Ok); +} + +TEST_F(FlatTensorDataMapTest, FlatTensorDataMap_GetMetadata) { + Result data_map = + FlatTensorDataMap::load(data_map_loader_.get()); + EXPECT_EQ(data_map.error(), Error::Ok); + + // Check tensor layouts are correct. + // From //executorch/test/models/linear_model.py, we have the tensors + // self.a = 3 * torch.ones(2, 2, dtype=torch.float) + // self.b = 2 * torch.ones(2, 2, dtype=torch.float) + Result const_a_res = data_map->get_metadata("a"); + ASSERT_EQ(Error::Ok, const_a_res.error()); + + const TensorLayout const_a = const_a_res.get(); + EXPECT_EQ(const_a.scalar_type(), executorch::aten::ScalarType::Float); + auto sizes_a = const_a.sizes(); + EXPECT_EQ(sizes_a.size(), 2); + EXPECT_EQ(sizes_a[0], 2); + EXPECT_EQ(sizes_a[1], 2); + auto dim_order_a = const_a.dim_order(); + EXPECT_EQ(dim_order_a.size(), 2); + EXPECT_EQ(dim_order_a[0], 0); + EXPECT_EQ(dim_order_a[1], 1); + + Result const_b_res = data_map->get_metadata("b"); + ASSERT_EQ(Error::Ok, const_b_res.error()); + + const TensorLayout const_b = const_b_res.get(); + EXPECT_EQ(const_b.scalar_type(), executorch::aten::ScalarType::Float); + auto sizes_b = const_b.sizes(); + EXPECT_EQ(sizes_b.size(), 2); + EXPECT_EQ(sizes_b[0], 2); + EXPECT_EQ(sizes_b[1], 2); + auto dim_order_b = const_b.dim_order(); + EXPECT_EQ(dim_order_b.size(), 2); + EXPECT_EQ(dim_order_b[0], 0); + EXPECT_EQ(dim_order_b[1], 1); + + // Check get_metadata fails when key is not found. + Result const_c_res = data_map->get_metadata("c"); + EXPECT_EQ(const_c_res.error(), Error::NotFound); +} + +TEST_F(FlatTensorDataMapTest, FlatTensorDataMap_GetData) { + Result data_map = + FlatTensorDataMap::load(data_map_loader_.get()); + EXPECT_EQ(data_map.error(), Error::Ok); + + // Check tensor data sizes are correct. + Result data_a_res = data_map->get_data("a"); + ASSERT_EQ(Error::Ok, data_a_res.error()); + FreeableBuffer data_a = std::move(data_a_res.get()); + EXPECT_EQ(data_a.size(), 16); + + Result data_b_res = data_map->get_data("b"); + ASSERT_EQ(Error::Ok, data_b_res.error()); + FreeableBuffer data_b = std::move(data_b_res.get()); + EXPECT_EQ(data_b.size(), 16); + + // Check get_data fails when key is not found. + Result data_c_res = data_map->get_data("c"); + EXPECT_EQ(data_c_res.error(), Error::NotFound); +} + +TEST_F(FlatTensorDataMapTest, FlatTensorDataMap_Keys) { + Result data_map = + FlatTensorDataMap::load(data_map_loader_.get()); + EXPECT_EQ(data_map.error(), Error::Ok); + + // Check num tensors is 2. + Result num_tensors_res = data_map->get_num_keys(); + ASSERT_EQ(Error::Ok, num_tensors_res.error()); + EXPECT_EQ(num_tensors_res.get(), 2); + + // Check get_key returns the correct keys. + Result key0_res = data_map->get_key(0); + ASSERT_EQ(Error::Ok, key0_res.error()); + EXPECT_EQ(strcmp(key0_res.get(), "a"), 0); + + Result key1_res = data_map->get_key(1); + ASSERT_EQ(Error::Ok, key1_res.error()); + EXPECT_EQ(strcmp(key1_res.get(), "b"), 0); + + // Check get_key fails when out of bounds. + Result key2_res = data_map->get_key(2); + EXPECT_EQ(key2_res.error(), Error::InvalidArgument); +} diff --git a/extension/flat_tensor/test/targets.bzl b/extension/flat_tensor/test/targets.bzl index fe845e31f5b..813a1b56d98 100644 --- a/extension/flat_tensor/test/targets.bzl +++ b/extension/flat_tensor/test/targets.bzl @@ -1,6 +1,6 @@ load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -def define_common_targets(): +def define_common_targets(is_fbcode=False): """Defines targets that should be shared between fbcode and xplat. The directory containing this targets.bzl file should also contain both @@ -29,3 +29,29 @@ def define_common_targets(): "//executorch/extension/tensor:tensor", ], ) + + if not runtime.is_oss and is_fbcode: + modules_env = { + # The tests use this var to find the program file to load. This uses + # an fbcode target path because the authoring/export tools + # intentionally don't work in xplat (since they're host-only tools). + "ET_MODULE_LINEAR_PROGRAM": "$(location fbcode//executorch/test/models:exported_programs_with_data_separated[ModuleLinear.pte])", + "ET_MODULE_LINEAR_DATA": "$(location fbcode//executorch/test/models:exported_programs_with_data_separated[ModuleLinear.ptd])", + } + + runtime.cxx_test( + name = "flat_tensor_data_map", + srcs = [ + "flat_tensor_data_map_test.cpp", + ], + deps = [ + "//executorch/extension/data_loader:file_data_loader", + "//executorch/extension/flat_tensor:flat_tensor_data_map", + "//executorch/extension/flat_tensor/serialize:flat_tensor_header", + "//executorch/extension/flat_tensor/serialize:generated_headers", + "//executorch/extension/flat_tensor/serialize:schema", + "//executorch/runtime/core:named_data_map", + "//executorch/runtime/core/exec_aten:lib", + ], + env = modules_env, + ) diff --git a/runtime/core/data_loader.h b/runtime/core/data_loader.h index 876212de8fb..45fd1bc8189 100644 --- a/runtime/core/data_loader.h +++ b/runtime/core/data_loader.h @@ -48,6 +48,10 @@ class DataLoader { * Data used for initializing mutable tensors. */ Mutable, + /** + * Data used for initializing external tensors. + */ + External, }; /// Type of the segment.