diff --git a/exir/version.py b/exir/version.py index 9f84c3abf24..389a8a8a350 100644 --- a/exir/version.py +++ b/exir/version.py @@ -6,4 +6,12 @@ # pyre-strict +# Stamped into Program.version of every exported PTE file. Keep in sync with +# Program::kMaxSupportedSchemaVersion in //executorch/runtime/executor/program.h, +# which is the highest version the C++ runtime agrees to load. +# +# Bump this only for a change that an older runtime would misread if it went +# unnoticed: a semantic change to an existing field, or a new field the runtime +# must understand to execute correctly. Purely additive optional fields stay +# backward/forward compatible (see schema/README.md) and need no bump. EXECUTORCH_SCHEMA_VERSION = 0 diff --git a/extension/flat_tensor/flat_tensor_data_map.cpp b/extension/flat_tensor/flat_tensor_data_map.cpp index 342e29e63fc..288c5fa6689 100644 --- a/extension/flat_tensor/flat_tensor_data_map.cpp +++ b/extension/flat_tensor/flat_tensor_data_map.cpp @@ -293,6 +293,21 @@ ET_NODISCARD Result FlatTensorDataMap::get_key( const flat_tensor_flatbuffer::FlatTensor* flat_tensor = flat_tensor_flatbuffer::GetFlatTensor(flat_tensor_data->data()); + // The file identifier above ("FT01") is bumped by convention only on a + // backward-incompatible schema change, so it selects a schema family. The + // version is the finer gate within that family: a file written by a newer + // exporter is refused here instead of being misread field by field. Older + // files stay loadable because the schema only grows by appending optional + // fields. + ET_CHECK_OR_RETURN_ERROR( + flat_tensor->version() <= kMaxSupportedSchemaVersion, + InvalidExternalData, + "FlatTensor schema version %u is newer than the highest this runtime " + "supports (%u). Export the data with an older ExecuTorch, or update the " + "runtime.", + flat_tensor->version(), + kMaxSupportedSchemaVersion); + // Validate flat_tensor. ET_CHECK_OR_RETURN_ERROR( flat_tensor->named_data() != nullptr, diff --git a/extension/flat_tensor/flat_tensor_data_map.h b/extension/flat_tensor/flat_tensor_data_map.h index 7b66eeab470..f0d5608e068 100644 --- a/extension/flat_tensor/flat_tensor_data_map.h +++ b/extension/flat_tensor/flat_tensor_data_map.h @@ -35,6 +35,17 @@ namespace extension { class FlatTensorDataMap final : public executorch::ET_RUNTIME_NAMESPACE::NamedDataMap { public: + /** + * The highest data schema version that this runtime can read. + * + * Keep in sync with kSchemaVersion in + * //executorch/extension/flat_tensor/serialize/serialize.h and with + * _FLAT_TENSOR_VERSION in + * //executorch/extension/flat_tensor/serialize/serialize.py, which are the + * versions that the two writers stamp into every PTD file. + */ + static constexpr uint32_t kMaxSupportedSchemaVersion = 0; + /** * Creates a new DataMap that wraps FlatTensor data. * diff --git a/extension/flat_tensor/serialize/serialize.h b/extension/flat_tensor/serialize/serialize.h index 759fc8d455b..694d2a3874c 100644 --- a/extension/flat_tensor/serialize/serialize.h +++ b/extension/flat_tensor/serialize/serialize.h @@ -19,6 +19,9 @@ namespace flat_tensor { /** * Schema version of the .ptd format. Should be kept in sync with serialize.py + * and with FlatTensorDataMap::kMaxSupportedSchemaVersion in + * //executorch/extension/flat_tensor/flat_tensor_data_map.h, which is the + * highest version the runtime agrees to load. */ constexpr uint32_t kSchemaVersion = 0; diff --git a/extension/flat_tensor/serialize/serialize.py b/extension/flat_tensor/serialize/serialize.py index 94303958caa..350fe96d92a 100644 --- a/extension/flat_tensor/serialize/serialize.py +++ b/extension/flat_tensor/serialize/serialize.py @@ -42,7 +42,10 @@ # Alignment of the flatbuffer (after the header). _FLATBUFFER_ALIGNMENT: int = 16 -# Current version. Keep in sync with c++ version number in serialize. +# Current version. Keep in sync with c++ version number in serialize, and with +# FlatTensorDataMap::kMaxSupportedSchemaVersion in +# //executorch/extension/flat_tensor/flat_tensor_data_map.h, which is the +# highest version the runtime agrees to load. _FLAT_TENSOR_VERSION: int = 0 diff --git a/extension/flat_tensor/test/flat_tensor_data_map_test.cpp b/extension/flat_tensor/test/flat_tensor_data_map_test.cpp index 0a5709a5ad3..b747cf1a98b 100644 --- a/extension/flat_tensor/test/flat_tensor_data_map_test.cpp +++ b/extension/flat_tensor/test/flat_tensor_data_map_test.cpp @@ -18,14 +18,8 @@ #include using namespace ::testing; -using executorch::extension::BufferDataLoader; -using executorch::extension::FileDataLoader; -using executorch::extension::FlatTensorDataMap; -using executorch::runtime::DataLoader; -using executorch::runtime::Error; -using executorch::runtime::FreeableBuffer; -using executorch::runtime::Result; -using executorch::runtime::TensorLayout; +using namespace executorch::extension; +using namespace executorch::runtime; class FlatTensorDataMapTest : public ::testing::Test { protected: @@ -181,3 +175,94 @@ TEST_F(FlatTensorDataMapTest, LoadAndCheckSize) { FlatTensorDataMap::load(&truncated_loader); ASSERT_EQ(truncated_program.error(), Error::InvalidExternalData); } + +namespace { + +constexpr size_t kAlignment = 16; + +size_t aligned_up(size_t size) { + return (size + kAlignment - 1) & ~(kAlignment - 1); +} + +// Builds the smallest PTD file that FlatTensorDataMap::load() accepts, stamped +// with the given schema version and holding no data. Follows the layout written +// by save_ptd(): the header is embedded in the flatbuffer region, so the offset +// to the root table shifts by the size of the header. +std::vector CreateDataWithVersion(uint32_t version) { + flatbuffers::FlatBufferBuilder builder; + auto flat_tensor = flat_tensor_flatbuffer::CreateFlatTensor( + builder, + version, + builder.CreateVector( + std::vector< + flatbuffers::Offset>{}), + builder.CreateVector( + std::vector< + flatbuffers::Offset>{})); + builder.Finish(flat_tensor, flat_tensor_flatbuffer::FlatTensorIdentifier()); + + const uint8_t* flatbuffer = builder.GetBufferPointer(); + const size_t flatbuffer_size = builder.GetSize(); + const size_t header_size = + aligned_up(FlatTensorHeader::kHeaderExpectedLength); + + std::vector data; + auto append = [&data](const void* bytes, size_t size) { + const uint8_t* begin = static_cast(bytes); + data.insert(data.end(), begin, begin + size); + }; + + uint32_t root_table_offset = *reinterpret_cast(flatbuffer) + + static_cast(header_size); + append(&root_table_offset, sizeof(root_table_offset)); + append(flatbuffer + sizeof(root_table_offset), 4); // File identifier. + + append(FlatTensorHeader::kMagic, sizeof(FlatTensorHeader::kMagic)); + uint32_t header_length = FlatTensorHeader::kHeaderExpectedLength; + append(&header_length, sizeof(header_length)); + uint64_t header_fields[] = { + header_size, // Offset to the flatbuffer. + flatbuffer_size, + header_size + aligned_up(flatbuffer_size), // Offset to the segments. + 0, // Segment data size. + }; + append(header_fields, sizeof(header_fields)); + data.resize(sizeof(root_table_offset) + 4 + header_size, 0); + + // The first eight bytes of the flatbuffer were written above, before the + // header. + append(flatbuffer + 8, flatbuffer_size - 8); + data.resize(header_size + aligned_up(flatbuffer_size), 0); + + return data; +} + +} // namespace + +TEST_F(FlatTensorDataMapTest, SupportedSchemaVersionLoads) { + std::vector data = + CreateDataWithVersion(FlatTensorDataMap::kMaxSupportedSchemaVersion); + + alignas(16) uint8_t aligned_buffer[512]; + ASSERT_LE(data.size(), sizeof(aligned_buffer)); + memcpy(aligned_buffer, data.data(), data.size()); + + BufferDataLoader loader(aligned_buffer, data.size()); + Result data_map = FlatTensorDataMap::load(&loader); + + EXPECT_EQ(data_map.error(), Error::Ok); +} + +TEST_F(FlatTensorDataMapTest, NewerSchemaVersionFailsToLoad) { + std::vector data = + CreateDataWithVersion(FlatTensorDataMap::kMaxSupportedSchemaVersion + 1); + + alignas(16) uint8_t aligned_buffer[512]; + ASSERT_LE(data.size(), sizeof(aligned_buffer)); + memcpy(aligned_buffer, data.data(), data.size()); + + BufferDataLoader loader(aligned_buffer, data.size()); + Result data_map = FlatTensorDataMap::load(&loader); + + EXPECT_EQ(data_map.error(), Error::InvalidExternalData); +} diff --git a/runtime/executor/program.cpp b/runtime/executor/program.cpp index 987850ccbc1..ecc9186499c 100644 --- a/runtime/executor/program.cpp +++ b/runtime/executor/program.cpp @@ -226,6 +226,21 @@ Result get_execution_plan( const executorch_flatbuffer::Program* flatbuffer_program = executorch_flatbuffer::GetProgram(program_data->data()); + // The file identifier above ("ET12") is bumped by convention only on a + // backward-incompatible schema change, so it selects a schema family. The + // version is the finer gate within that family: a file written by a newer + // exporter is refused here instead of being misread field by field. Older + // files stay loadable because the schema only grows by appending optional + // fields. + ET_CHECK_OR_RETURN_ERROR( + flatbuffer_program->version() <= kMaxSupportedSchemaVersion, + InvalidProgram, + "Program schema version %u is newer than the highest this runtime " + "supports (%u). Export the model with an older ExecuTorch, or update " + "the runtime.", + flatbuffer_program->version(), + kMaxSupportedSchemaVersion); + // Instantiate PteDataMap if named_data is present. const auto named_data = flatbuffer_program->named_data(); std::optional pte_data_map = std::nullopt; diff --git a/runtime/executor/program.h b/runtime/executor/program.h index e1208e52454..613137c4983 100644 --- a/runtime/executor/program.h +++ b/runtime/executor/program.h @@ -75,6 +75,15 @@ class Program final { InternalConsistency, }; + /** + * The highest program schema version that this runtime can read. + * + * Keep in sync with EXECUTORCH_SCHEMA_VERSION in + * //executorch/exir/version.py, which is the version that the exporter stamps + * into every PTE file. + */ + static constexpr uint32_t kMaxSupportedSchemaVersion = 0; + /** * Loads a Program from the provided loader. The Program will hold a pointer * to the loader, which must outlive the returned Program instance. diff --git a/runtime/executor/test/program_test.cpp b/runtime/executor/test/program_test.cpp index 72308e6e8d7..544c4851c22 100644 --- a/runtime/executor/test/program_test.cpp +++ b/runtime/executor/test/program_test.cpp @@ -210,6 +210,96 @@ TEST_F(ProgramTest, BadMagicFailsToLoad) { } } +namespace { + +// Builds the smallest program that Program::load() accepts, stamped with the +// given schema version. +std::vector CreateProgramWithVersion(uint32_t version) { + flatbuffers::FlatBufferBuilder builder(1024); + + auto plan_name = builder.CreateString("forward"); + auto empty_values = builder.CreateVector( + std::vector>{}); + auto empty_inputs = builder.CreateVector(std::vector{}); + auto empty_outputs = builder.CreateVector(std::vector{}); + auto empty_chains = builder.CreateVector( + std::vector>{}); + auto empty_operators = builder.CreateVector( + std::vector>{}); + auto empty_delegates = builder.CreateVector( + std::vector< + flatbuffers::Offset>{}); + auto buffer_sizes = builder.CreateVector(std::vector{0}); + + auto execution_plan = executorch_flatbuffer::CreateExecutionPlan( + builder, + plan_name, + /*container_meta_type=*/0, + empty_values, + empty_inputs, + empty_outputs, + empty_chains, + empty_operators, + empty_delegates, + buffer_sizes); + auto execution_plans = builder.CreateVector( + std::vector>{ + execution_plan}); + + // A constant segment holding only the placeholder offset means "no + // constants", which keeps this program off the deprecated constant_buffer + // path that OSS builds compile out. + auto constant_segment = executorch_flatbuffer::CreateSubsegmentOffsets( + builder, + /*segment_index=*/0, + builder.CreateVector(std::vector{0})); + + auto program = executorch_flatbuffer::CreateProgram( + builder, + version, + execution_plans, + /*constant_buffer=*/0, + /*backend_delegate_data=*/0, + /*segments=*/0, + constant_segment); + builder.Finish(program, executorch_flatbuffer::ProgramIdentifier()); + + const uint8_t* data = builder.GetBufferPointer(); + return std::vector(data, data + builder.GetSize()); +} + +} // namespace + +TEST_F(ProgramTest, SupportedSchemaVersionLoads) { + std::vector data = + CreateProgramWithVersion(Program::kMaxSupportedSchemaVersion); + + alignas(16) uint8_t aligned_buffer[2048]; + ASSERT_LE(data.size(), sizeof(aligned_buffer)); + memcpy(aligned_buffer, data.data(), data.size()); + + BufferDataLoader data_loader(aligned_buffer, data.size()); + Result program = Program::load(&data_loader, kDefaultVerification); + + EXPECT_EQ(program.error(), Error::Ok); +} + +TEST_F(ProgramTest, NewerSchemaVersionFailsToLoad) { + std::vector data = + CreateProgramWithVersion(Program::kMaxSupportedSchemaVersion + 1); + + alignas(16) uint8_t aligned_buffer[2048]; + ASSERT_LE(data.size(), sizeof(aligned_buffer)); + memcpy(aligned_buffer, data.data(), data.size()); + + // Use minimal verification to show that even the cheapest level catches it. + BufferDataLoader data_loader(aligned_buffer, data.size()); + Result program = + Program::load(&data_loader, Program::Verification::Minimal); + + EXPECT_EQ(program.error(), Error::InvalidProgram); +} + // These tests require ET_ENABLE_PROGRAM_VERIFICATION to be enabled. // In Release builds, verification is disabled by default to save binary size. #ifndef ET_ENABLE_PROGRAM_VERIFICATION diff --git a/schema/test/test_schema.py b/schema/test/test_schema.py index 6e6a3006a2a..18077f1c6d6 100644 --- a/schema/test/test_schema.py +++ b/schema/test/test_schema.py @@ -8,6 +8,7 @@ import filecmp import os +import re import unittest @@ -47,6 +48,99 @@ def test_schema_sync(self) -> None: f"Please sync the schema by copying from {canonical_path}.", ) + def test_schema_version_constants_in_sync(self) -> None: + """The schema version constants must not silently drift apart. + + A PTE/PTD file's schema version is stamped by a writer and gated by a + runtime reader. The writer value and the reader ceiling live in separate + files, in two languages, and today are kept together only by comments. + If a writer is bumped without its reader (or the two PTD writers + disagree), a runtime will either reject a file it should read or read a + file it should reject. This test turns those comments into a check. + + The rule (see schema/README.md for the compatibility policy): + * A reader ceiling may be >= its writer version: a runtime is allowed + to support a version before any writer emits it. It must never be + lower, which would refuse files the matching writer produces. + * The two PTD writers (Python and C++) stamp the same field of the + same file, so they must be exactly equal. + * The PTE and PTD families are independent; no relation between them. + + The values are parsed as text rather than imported: importing the + Python writers pulls in torch, and the C++ constants have no Python + binding. Parsing keys on the file path, not the symbol name, because + the two reader ceilings share the name kMaxSupportedSchemaVersion. + """ + prefix = ( + "executorch/" if os.path.exists("executorch/schema/scalar_type.fbs") else "" + ) + + def read_constant(path: str, pattern: str) -> int: + full_path = prefix + path + with open(full_path) as f: + contents = f.read() + match = re.search(pattern, contents) + self.assertIsNotNone( + match, + f"Could not find the schema version constant in {full_path} " + f"(pattern {pattern!r}). If the declaration moved or was " + f"reformatted, update this test so the sync check keeps working.", + ) + # match is not None: asserted above. + return int(match.group(1)) # pyre-ignore[16] + + # PTE (program) family. + pte_writer = read_constant( + "exir/version.py", + r"EXECUTORCH_SCHEMA_VERSION\s*=\s*(\d+)", + ) + pte_reader = read_constant( + "runtime/executor/program.h", + r"kMaxSupportedSchemaVersion\s*=\s*(\d+)", + ) + + # PTD (data) family. Two writers stamp the same field; one reader gate. + ptd_writer_py = read_constant( + "extension/flat_tensor/serialize/serialize.py", + r"_FLAT_TENSOR_VERSION\s*:\s*int\s*=\s*(\d+)", + ) + ptd_writer_cpp = read_constant( + "extension/flat_tensor/serialize/serialize.h", + r"kSchemaVersion\s*=\s*(\d+)", + ) + ptd_reader = read_constant( + "extension/flat_tensor/flat_tensor_data_map.h", + r"kMaxSupportedSchemaVersion\s*=\s*(\d+)", + ) + + # A writer must never stamp a version its own runtime reader refuses. + self.assertLessEqual( + pte_writer, + pte_reader, + "EXECUTORCH_SCHEMA_VERSION (exir/version.py) is higher than " + "Program::kMaxSupportedSchemaVersion (runtime/executor/program.h). " + "The exporter would stamp PTE files this runtime refuses. Raise the " + "runtime ceiling before (or with) the exporter version.", + ) + + # The two PTD writers stamp the same field of the same file. + self.assertEqual( + ptd_writer_py, + ptd_writer_cpp, + "_FLAT_TENSOR_VERSION (serialize.py) and kSchemaVersion " + "(serialize.h) disagree. Both writers stamp the same version into " + "every PTD file and must be equal.", + ) + self.assertLessEqual( + ptd_writer_cpp, + ptd_reader, + "The PTD writers are higher than " + "FlatTensorDataMap::kMaxSupportedSchemaVersion " + "(flat_tensor_data_map.h). The writers would stamp PTD files this " + "runtime refuses. Raise the runtime ceiling before (or with) the " + "writers.", + ) + if __name__ == "__main__": unittest.main()