From 36f67da75e05d21309022b42e1189b18171fa05f Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Wed, 16 Sep 2026 13:10:33 +0900 Subject: [PATCH 1/4] feat(import): cook HDR environment maps --- ZEngine/ZEngine/Importers/AssetCodec.cpp | 80 +++++++-- ZEngine/ZEngine/Importers/AssetCodec.h | 56 ++++++- .../Importers/EnvironmentMapImporter.cpp | 111 +++++++++++-- .../Importers/EnvironmentMapImporter.h | 9 +- .../ZEngine/Importers/ImportCoordinator.cpp | 8 + ZEngine/ZEngine/Importers/TextureImporter.h | 4 +- ZEngine/ZEngine/Rendering/Buffers/Bitmap.cpp | 73 +++++++++ ZEngine/ZEngine/Rendering/Buffers/Bitmap.h | 3 + .../Rendering/RenderResourceManager.cpp | 57 ++----- .../Rendering/Renderers/GraphicRenderer.cpp | 58 ++++++- .../Rendering/Scenes/SkyEnvironment.cpp | 105 +++++++----- .../ZEngine/Rendering/Scenes/SkyEnvironment.h | 25 ++- ZEngine/docs/future-plan/sky-rendering.md | 8 +- .../Rendering/EnvironmentMapCooking_test.cpp | 155 ++++++++++++++++++ .../tests/Rendering/SkyEnvironment_test.cpp | 38 +++++ .../tests/Rendering/TextureImporterTest.cpp | 6 +- 16 files changed, 647 insertions(+), 149 deletions(-) create mode 100644 ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp diff --git a/ZEngine/ZEngine/Importers/AssetCodec.cpp b/ZEngine/ZEngine/Importers/AssetCodec.cpp index 1bf77de6e..9da7ea850 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.cpp +++ b/ZEngine/ZEngine/Importers/AssetCodec.cpp @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #include #include @@ -324,6 +326,35 @@ namespace ZEngine::Importers::AssetCodec return true; } + uint32_t GetEnvironmentMapFullMipCount(uint32_t face_size) + { + uint32_t mip_count = 1; + while (face_size > 1) + { + face_size >>= 1; + ++mip_count; + } + return mip_count; + } + + bool IsEnvironmentMapFileHeaderValid(const EnvironmentMapFileHeader& header) + { + if (header.MagicNumber != ZENVMAP_MAGIC || header.Version != ENVIRONMENT_MAP_FILE_VERSION || header.HeaderByteSize != sizeof(EnvironmentMapFileHeader) || header.ImporterVersion == 0 || header.FaceWidth == 0 || header.FaceWidth != header.FaceHeight || header.FaceWidth > ENVIRONMENT_MAP_MAX_FACE_SIZE || header.Channel != 4 || header.LayerCount != 6 || header.MipCount != GetEnvironmentMapFullMipCount(header.FaceWidth) || header.ColorSpace != static_cast(EnvironmentMapColorSpace::LinearScene) || + header.Orientation != static_cast(EnvironmentMapOrientation::RendererCanonical) || header.MipPolicy != static_cast(EnvironmentMapMipPolicy::GenerateOnGpu) || !std::isfinite(header.Exposure) || header.Exposure <= 0.0f) + return false; + + constexpr uint64_t k_bytes_per_pixel = sizeof(float) * 4; + const uint64_t face_pixels = static_cast(header.FaceWidth) * static_cast(header.FaceHeight); + if (face_pixels > std::numeric_limits::max() / header.LayerCount || face_pixels * header.LayerCount > std::numeric_limits::max() / k_bytes_per_pixel) + return false; + return header.BufferByteSize == face_pixels * header.LayerCount * k_bytes_per_pixel; + } + + bool DoesEnvironmentMapHeaderMatchSource(const EnvironmentMapFileHeader& header, uint64_t source_hash) + { + return IsEnvironmentMapFileHeaderValid(header) && header.SourceHash == source_hash; + } + bool DeserializeEnvironmentMapFile(const char* zenvmap_file, Rendering::Buffers::Bitmap& out_cubemap) { std::ifstream in(zenvmap_file, std::ios::binary); @@ -332,12 +363,22 @@ namespace ZEngine::Importers::AssetCodec EnvironmentMapFileHeader header{}; in.read(reinterpret_cast(&header), sizeof(header)); - if (!in.good() || header.MagicNumber != ZENVMAP_MAGIC) + if (!in.good() || !IsEnvironmentMapFileHeaderValid(header)) + return false; + + in.seekg(0, std::ios::end); + const std::streamoff expected_file_size = static_cast(header.HeaderByteSize) + static_cast(header.BufferByteSize); + if (in.tellg() != expected_file_size) + return false; + + in.seekg(static_cast(header.HeaderByteSize), std::ios::beg); + Rendering::Buffers::Bitmap cubemap = Rendering::Buffers::Bitmap::Create(static_cast(header.FaceWidth), static_cast(header.FaceHeight), static_cast(header.LayerCount), static_cast(header.Channel), Rendering::Buffers::BitmapFormat::Float, Rendering::Buffers::BitmapType::CubeMap); + in.read(reinterpret_cast(cubemap.Buffer), static_cast(header.BufferByteSize)); + if (!in.good()) return false; - out_cubemap = Rendering::Buffers::Bitmap::Create(header.FaceWidth, header.FaceHeight, header.LayerCount, header.Channel, Rendering::Buffers::BitmapFormat::Float, Rendering::Buffers::BitmapType::CubeMap); - in.read(reinterpret_cast(out_cubemap.Buffer), static_cast(header.BufferByteSize)); - return in.good(); + out_cubemap = std::move(cubemap); + return true; } bool ReadEnvironmentMapFileHeader(const char* zenvmap_file, EnvironmentMapFileHeader& out_header) @@ -346,11 +387,14 @@ namespace ZEngine::Importers::AssetCodec if (!in.is_open()) return false; in.read(reinterpret_cast(&out_header), sizeof(EnvironmentMapFileHeader)); - return in.good() && (out_header.MagicNumber == ZENVMAP_MAGIC); + return in.good() && IsEnvironmentMapFileHeaderValid(out_header); } - Core::VFS::VFSResult SerializeEnvironmentMapFileVFS(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, const Rendering::Buffers::Bitmap& cubemap) + Core::VFS::VFSResult SerializeEnvironmentMapFileVFS(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, const Rendering::Buffers::Bitmap& cubemap, const EnvironmentMapCookMetadata& metadata) { + if (cubemap.Type != Rendering::Buffers::BitmapType::CubeMap || cubemap.Width <= 0 || cubemap.Width != cubemap.Height || cubemap.Width > static_cast(ENVIRONMENT_MAP_MAX_FACE_SIZE) || cubemap.Channel != 4 || cubemap.Layers != 6 || cubemap.Format != Rendering::Buffers::BitmapFormat::Float || !cubemap.Buffer || !std::isfinite(metadata.Exposure) || metadata.Exposure <= 0.0f || metadata.ImporterVersion == 0) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::InvalidPath); + // Build .tmp path for atomic write char tmp_buf[MAX_FILE_PATH_COUNT] = {}; const char* raw = out_path.CStr(); @@ -368,14 +412,24 @@ namespace ZEngine::Importers::AssetCodec Core::VFS::IVFSFile* file = open_result.Value(); EnvironmentMapFileHeader header{ - .MagicNumber = ZENVMAP_MAGIC, - .Version = ASSET_FILE_VERSION, - .FaceWidth = cubemap.Width, - .FaceHeight = cubemap.Height, - .Channel = cubemap.Channel, - .LayerCount = cubemap.Layers, - .BufferByteSize = static_cast(cubemap.BufferSize), + .MagicNumber = ZENVMAP_MAGIC, + .Version = ENVIRONMENT_MAP_FILE_VERSION, + .HeaderByteSize = sizeof(EnvironmentMapFileHeader), + .ImporterVersion = metadata.ImporterVersion, + .SourceHash = metadata.SourceHash, + .FaceWidth = static_cast(cubemap.Width), + .FaceHeight = static_cast(cubemap.Height), + .Channel = static_cast(cubemap.Channel), + .LayerCount = static_cast(cubemap.Layers), + .MipCount = GetEnvironmentMapFullMipCount(static_cast(cubemap.Width)), + .ColorSpace = static_cast(EnvironmentMapColorSpace::LinearScene), + .Orientation = static_cast(EnvironmentMapOrientation::RendererCanonical), + .MipPolicy = static_cast(EnvironmentMapMipPolicy::GenerateOnGpu), + .Exposure = metadata.Exposure, + .BufferByteSize = static_cast(cubemap.BufferSize), }; + if (!IsEnvironmentMapFileHeaderValid(header)) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::InvalidPath); const auto* hdr_bytes = reinterpret_cast(&header); auto w1 = file->Write({hdr_bytes, sizeof(header)}, 0); diff --git a/ZEngine/ZEngine/Importers/AssetCodec.h b/ZEngine/ZEngine/Importers/AssetCodec.h index b985f7b01..c3930c807 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.h +++ b/ZEngine/ZEngine/Importers/AssetCodec.h @@ -9,6 +9,25 @@ namespace ZEngine::Importers::AssetCodec { + inline constexpr uint32_t ENVIRONMENT_MAP_FILE_VERSION = 2; + inline constexpr uint32_t ENVIRONMENT_MAP_IMPORTER_VERSION = 1; + inline constexpr uint32_t ENVIRONMENT_MAP_MAX_FACE_SIZE = 1024; + + enum class EnvironmentMapColorSpace : uint32_t + { + LinearScene = 1, + }; + + enum class EnvironmentMapOrientation : uint32_t + { + RendererCanonical = 1, + }; + + enum class EnvironmentMapMipPolicy : uint32_t + { + GenerateOnGpu = 1, + }; + // Binary codec for ZEngine's on-disk asset formats (.zasset, .zematerial, .zetextures, .zenvmap). // These are the cook-time serialization helpers used by format importers to produce // the cooked binary artifacts that AssetManager loads at runtime. @@ -46,13 +65,30 @@ namespace ZEngine::Importers::AssetCodec struct EnvironmentMapFileHeader { - uint32_t MagicNumber = 0; - uint32_t Version = 0; - int32_t FaceWidth = 0; - int32_t FaceHeight = 0; - int32_t Channel = 0; - int32_t LayerCount = 0; - uint64_t BufferByteSize = 0; + uint32_t MagicNumber = 0; + uint32_t Version = 0; + uint32_t HeaderByteSize = 0; + uint32_t ImporterVersion = 0; + uint64_t SourceHash = 0; + uint32_t FaceWidth = 0; + uint32_t FaceHeight = 0; + uint32_t Channel = 0; + uint32_t LayerCount = 0; + uint32_t MipCount = 0; + uint32_t ColorSpace = 0; + uint32_t Orientation = 0; + uint32_t MipPolicy = 0; + float Exposure = 1.0f; + uint32_t Reserved = 0; + uint64_t BufferByteSize = 0; + }; + static_assert(sizeof(EnvironmentMapFileHeader) == 72, "Environment-map artifact header must remain stable"); + + struct EnvironmentMapCookMetadata + { + uint64_t SourceHash = 0; + float Exposure = 1.0f; + uint32_t ImporterVersion = ENVIRONMENT_MAP_IMPORTER_VERSION; }; AssetImporterOutput SerializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, AssetMesh& mesh, AssetNodeHierarchy& hierarchies, const ImportConfiguration& config); @@ -61,9 +97,13 @@ namespace ZEngine::Importers::AssetCodec AssetImporterOutput SerializeTextureAssetFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::ArrayView textures, const ImportConfiguration& config); + [[nodiscard]] uint32_t GetEnvironmentMapFullMipCount(uint32_t face_size); + [[nodiscard]] bool IsEnvironmentMapFileHeaderValid(const EnvironmentMapFileHeader& header); + [[nodiscard]] bool DoesEnvironmentMapHeaderMatchSource(const EnvironmentMapFileHeader& header, uint64_t source_hash); + // VFS-based — writes through IVFSContext using atomic .tmp → rename protocol. // out_path: the VFS path to write (e.g. project://_cache/envmaps/.zenvmap) - Core::VFS::VFSResult SerializeEnvironmentMapFileVFS(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, const Rendering::Buffers::Bitmap& cubemap); + Core::VFS::VFSResult SerializeEnvironmentMapFileVFS(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, const Rendering::Buffers::Bitmap& cubemap, const EnvironmentMapCookMetadata& metadata = {}); void DeserializeMeshAssetFile(Core::Memory::ArenaAllocator* arena, const char* asset_file, AssetMesh& mesh, AssetNodeHierarchy& hierarchies); diff --git a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp index fa2486aba..e697df1c3 100644 --- a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp +++ b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp @@ -1,12 +1,14 @@ +#include #include #include #include #include #include +#include #include -#include #include -#include +#include +#include // stb_image implementation is defined once in RenderResourceManager.cpp. #include @@ -15,6 +17,18 @@ using namespace ZEngine::Rendering::Buffers; namespace ZEngine::Importers { + namespace + { + void AddImportSetting(Core::VFS::MetaFileData& meta, const char* key, const char* value) + { + if (meta.SettingsCount >= Core::VFS::META_MAX_SETTINGS) + return; + Core::VFS::MetaKeyValuePair& setting = meta.Settings[meta.SettingsCount++]; + std::snprintf(setting.Key, sizeof(setting.Key), "%s", key); + std::snprintf(setting.Value, sizeof(setting.Value), "%s", value); + } + } // namespace + void EnvironmentMapImporter::Initialize(Core::Memory::ArenaAllocator* arena) { arena->CreateSubArena(ZMega(32), &Arena); @@ -24,34 +38,74 @@ namespace ZEngine::Importers { if (!extension) return false; - return Helpers::secure_strcmp(extension, "hdr") == 0 || Helpers::secure_strcmp(extension, "exr") == 0; + // stb_image does not decode EXR. Do not advertise it until an EXR-capable + // importer is implemented and covered by the same artifact contract. + return Helpers::secure_strcmp(extension, "hdr") == 0; + } + + bool EnvironmentMapImporter::IsSupportedEquirectangularSource(int width, int height, const float* rgba_pixels) + { + if (!rgba_pixels || width <= 0 || height <= 0 || width != height * 2 || width % 4 != 0) + return false; + + const int face_size = width / 4; + if (face_size <= 0 || face_size > static_cast(AssetCodec::ENVIRONMENT_MAP_MAX_FACE_SIZE)) + return false; + + const size_t component_count = static_cast(width) * static_cast(height) * 4; + for (size_t index = 0; index < component_count; ++index) + if (!std::isfinite(rgba_pixels[index]) || rgba_pixels[index] < 0.0f) + return false; + return true; + } + + bool EnvironmentMapImporter::BuildArtifactPath(const uuids::uuid& asset_uuid, char* out_path, size_t out_path_size) + { + if (asset_uuid.is_nil() || !out_path || out_path_size == 0) + return false; + const int written = std::snprintf(out_path, out_path_size, "/_cache/envmaps/%s.zenvmap", uuids::to_string(asset_uuid).c_str()); + return written > 0 && static_cast(written) < out_path_size; } Core::VFS::VFSResult EnvironmentMapImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { - // Resolve to native path — stb_image works on the filesystem, not the VFS. - char native[MAX_FILE_PATH_COUNT] = {}; - path.ToNative(native, sizeof(native)); + // stb_image works on the filesystem, while the source identity and cooked + // artifact remain VFS paths. Resolve the source relative to its workspace. + char native[MAX_FILE_PATH_COUNT] = {}; + const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; + if (working_space && working_space[0] != '\0') + path.ResolveNative(working_space, native, sizeof(native)); + else + path.ToNative(native, sizeof(native)); int width = 0, height = 0, channel = 0; - const float* image_data = stbi_loadf(native, &width, &height, &channel, 4); + const float* image_data = stbi_loadf(native, &width, &height, &channel, STBI_rgb_alpha); if (!image_data) { ZENGINE_CORE_ERROR("EnvironmentMapImporter: failed to load '{}': {}", native, stbi_failure_reason()) return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); } + if (!IsSupportedEquirectangularSource(width, height, image_data)) + { + stbi_image_free(const_cast(image_data)); + ZENGINE_CORE_ERROR("EnvironmentMapImporter: '{}' must be a finite, non-negative 2:1 HDR equirectangular image with a face size no larger than {}", native, AssetCodec::ENVIRONMENT_MAP_MAX_FACE_SIZE) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::InvalidPath); + } + Core::Memory::TLSFSlab* slab = Helpers::GetWorkerSlab(); - Bitmap equirect = Bitmap::FromData(width, height, 1, 4, BitmapFormat::Float, BitmapType::Texture2D, image_data, slab); + Bitmap equirect = Bitmap::FromData(width, height, 1, STBI_rgb_alpha, BitmapFormat::Float, BitmapType::Texture2D, image_data); stbi_image_free(const_cast(image_data)); - Bitmap vertical_cross = BitmapConvert::EquirectToCross(equirect, slab); - Bitmap cubemap = BitmapConvert::CrossToCubemap(vertical_cross, slab); + Bitmap cubemap = BitmapConvert::EquirectToCubemap(equirect, slab); - // Write to project://_cache/envmaps/.zenvmap via VFS. - // Keyed by UUID — regenerable, gitignored, transparent to game code. + // The cache is UUID keyed, regenerable, and never stored in scene data. char vfs_path_buf[MAX_FILE_PATH_COUNT] = {}; - std::snprintf(vfs_path_buf, sizeof(vfs_path_buf), "/_cache/envmaps/%s.zenvmap", uuids::to_string(meta.AssetUUID).c_str()); + if (!BuildArtifactPath(meta.AssetUUID, vfs_path_buf, sizeof(vfs_path_buf))) + { + ZENGINE_CORE_ERROR("EnvironmentMapImporter: cannot build a cache path for '{}'", native) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::InvalidPath); + } auto out_path_result = Core::VFS::VFSPath::Parse(vfs_path_buf); if (!out_path_result.Succeeded()) @@ -61,16 +115,41 @@ namespace ZEngine::Importers } // Ensure the cache directory exists - auto cache_dir = Core::VFS::VFSPath::Parse("/_cache/envmaps").Value(); - ctx.CreateDir(cache_dir); // no-op if already exists + auto cache_dir = Core::VFS::VFSPath::Parse("/_cache/envmaps").Value(); + const auto create_directory_result = ctx.CreateDir(cache_dir); + if (create_directory_result.Failed() && create_directory_result.Error() != Core::VFS::VFSError::AlreadyExists) + { + ZENGINE_CORE_ERROR("EnvironmentMapImporter: cannot create the cache directory for '{}'", native) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); + } - auto write_result = AssetCodec::SerializeEnvironmentMapFileVFS(ctx, out_path_result.Value(), cubemap); + const AssetCodec::EnvironmentMapCookMetadata cook_metadata = {.SourceHash = meta.SourceHash}; + auto write_result = AssetCodec::SerializeEnvironmentMapFileVFS(ctx, out_path_result.Value(), cubemap, cook_metadata); if (write_result.Failed()) { ZENGINE_CORE_ERROR("EnvironmentMapImporter: failed to write .zenvmap for '{}'", native) return write_result; } + // Keep the artifact metadata beside the stable source UUID. The render + // path reads the cooked path and validates this source hash before upload. + Core::VFS::MetaFileData cooked_meta = meta; + std::snprintf(cooked_meta.ImporterName, sizeof(cooked_meta.ImporterName), "%s", "EnvironmentMapImporter"); + std::snprintf(cooked_meta.SourcePath, sizeof(cooked_meta.SourcePath), "%s", native); + std::snprintf(cooked_meta.ArtifactPath, sizeof(cooked_meta.ArtifactPath), "%s", vfs_path_buf); + cooked_meta.SettingsCount = 0; + AddImportSetting(cooked_meta, "artifact_version", "2"); + AddImportSetting(cooked_meta, "pixel_format", "rgba32f"); + AddImportSetting(cooked_meta, "color_space", "linear_scene"); + AddImportSetting(cooked_meta, "orientation", "renderer_canonical_v1"); + AddImportSetting(cooked_meta, "mip_policy", "generate_on_gpu"); + AddImportSetting(cooked_meta, "exposure", "1.0"); + if (Core::VFS::MetaFileIO::Write(ctx, path, cooked_meta).Failed()) + { + ZENGINE_CORE_ERROR("EnvironmentMapImporter: failed to write metadata for '{}'", native) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); + } + ZENGINE_CORE_INFO("EnvironmentMapImporter: cooked '{}' → '{}'", native, vfs_path_buf) return Core::VFS::VFSResult::Ok(); } diff --git a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.h b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.h index e41a1145c..d8e473293 100644 --- a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.h +++ b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.h @@ -4,9 +4,9 @@ namespace ZEngine::Importers { - // Imports .hdr and .exr equirectangular images, converts them to cubemaps, + // Imports .hdr equirectangular images, converts them to cubemaps, // and writes a .zenvmap cooked artifact. Registered with ImportCoordinator - // so the standard Enqueue/Tick pipeline handles all HDR/EXR files. + // so the standard Enqueue/Tick pipeline handles HDRI assets off the render thread. class EnvironmentMapImporter : public IAssetImporter { public: @@ -19,6 +19,11 @@ namespace ZEngine::Importers bool CanImport(const char* extension) const override; Core::VFS::VFSResult Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) override; + /// @brief Validates the source contract before conversion allocates its working buffers. + [[nodiscard]] static bool IsSupportedEquirectangularSource(int width, int height, const float* rgba_pixels); + /// @brief Builds the cache-only VFS path for a stable source asset UUID. + [[nodiscard]] static bool BuildArtifactPath(const uuids::uuid& asset_uuid, char* out_path, size_t out_path_size); + Core::Memory::ArenaAllocator Arena = {}; }; } // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/ImportCoordinator.cpp b/ZEngine/ZEngine/Importers/ImportCoordinator.cpp index 71a416592..57faf7918 100644 --- a/ZEngine/ZEngine/Importers/ImportCoordinator.cpp +++ b/ZEngine/ZEngine/Importers/ImportCoordinator.cpp @@ -146,7 +146,15 @@ namespace ZEngine::Importers { ZENGINE_CORE_INFO("[ImportCoordinator] Imported '{}'", job.Path.CStr()) if (coordinator->m_registry) + { + // Importers may have published a new cooked artifact path or + // versioned import metadata. Refresh the lightweight registry + // snapshot before notifying runtime consumers that the source is ready. + auto updated_meta = Core::VFS::MetaFileIO::Read(*coordinator->m_vfs_ctx, job.Path); + if (updated_meta.Succeeded()) + coordinator->m_registry->UpdateMeta(job.Meta.AssetUUID, updated_meta.Value(), Core::VFS::AssetState::Loaded); coordinator->m_registry->SetState(job.Meta.AssetUUID, Core::VFS::AssetState::Loaded); + } job.Callback.Invoke(true); coordinator->m_completed.value.fetch_add(1, std::memory_order_relaxed); } diff --git a/ZEngine/ZEngine/Importers/TextureImporter.h b/ZEngine/ZEngine/Importers/TextureImporter.h index cb320001a..c856801a7 100644 --- a/ZEngine/ZEngine/Importers/TextureImporter.h +++ b/ZEngine/ZEngine/Importers/TextureImporter.h @@ -4,8 +4,8 @@ namespace ZEngine::Importers { /// @brief Imports flat 2D raster textures (png/jpg/jpeg/bmp/tga/gif/psd/pic). - /// @details Does not claim hdr/exr (EnvironmentMapImporter's domain) or ktx/ktx2 - /// (not decodable by stb_image today). + /// @details Does not claim HDR environment sources or ktx/ktx2, which are + /// not flat stb_image texture inputs. class TextureImporter : public IAssetImporter { public: diff --git a/ZEngine/ZEngine/Rendering/Buffers/Bitmap.cpp b/ZEngine/ZEngine/Rendering/Buffers/Bitmap.cpp index b012e7c47..c11e55fb9 100644 --- a/ZEngine/ZEngine/Rendering/Buffers/Bitmap.cpp +++ b/ZEngine/ZEngine/Rendering/Buffers/Bitmap.cpp @@ -253,6 +253,79 @@ namespace ZEngine::Rendering::Buffers return out; } + Bitmap EquirectToCubemap(const Bitmap& input, Core::Memory::TLSFSlab* slab) + { + if (input.Type != BitmapType::Texture2D || input.Width <= 0 || input.Height <= 0 || input.Width != input.Height * 2 || input.Width % 4 != 0 || input.Channel <= 0 || !input.Buffer) + return Bitmap(); + + const int face_size = input.Width / 4; + Bitmap out = Bitmap::Create(face_size, face_size, 6, input.Channel, input.Format, BitmapType::CubeMap, slab); + const int cw = input.Width - 1; + const int ch = input.Height - 1; + // Preserve the face order and rotations produced by the legacy + // equirectangular -> vertical-cross -> cubemap conversion. Existing + // sky shaders and cached assets use this as the canonical order. + constexpr int k_source_faces[6] = {1, 3, 4, 5, 0, 2}; + constexpr bool k_flip_face[6] = {false, false, true, true, true, false}; + + const auto write_cubemap_pixel = [&out, face_size](int face, int x, int y, const Core::Maths::Vec4f& pixel) { + const size_t offset = static_cast(out.Channel) * (static_cast(face) * face_size * face_size + static_cast(y) * face_size + x); + if (out.Format == BitmapFormat::UnsignedByte) + { + if (out.Channel > 0) + out.Buffer[offset + 0] = uint8_t(pixel.x * 255.0f); + if (out.Channel > 1) + out.Buffer[offset + 1] = uint8_t(pixel.y * 255.0f); + if (out.Channel > 2) + out.Buffer[offset + 2] = uint8_t(pixel.z * 255.0f); + if (out.Channel > 3) + out.Buffer[offset + 3] = uint8_t(pixel.w * 255.0f); + } + else if (out.Format == BitmapFormat::Float) + { + float* const data = reinterpret_cast(out.Buffer); + if (out.Channel > 0) + data[offset + 0] = pixel.x; + if (out.Channel > 1) + data[offset + 1] = pixel.y; + if (out.Channel > 2) + data[offset + 2] = pixel.z; + if (out.Channel > 3) + data[offset + 3] = pixel.w; + } + }; + + for (int face = 0; face < 6; ++face) + { + for (int i = 0; i < face_size; ++i) + { + for (int j = 0; j < face_size; ++j) + { + const int source_i = k_flip_face[face] ? face_size - (i + 1) : i; + const int source_j = k_flip_face[face] ? face_size - (j + 1) : j; + const Core::Maths::Vec3f P = FaceCoordToXYZ(source_i, source_j, k_source_faces[face], face_size); + const float R = hypot(P.x, P.y); + const float theta = atan2(P.y, P.x); + const float phi = atan2(P.z, R); + const float Uf = float(2.0f * face_size * (theta + Core::Maths::PI) / Core::Maths::PI); + const float Vf = float(2.0f * face_size * (Core::Maths::PI / 2.0f - phi) / Core::Maths::PI); + const int U1 = Core::Maths::clamp(int(floor(Uf)), 0, cw); + const int V1 = Core::Maths::clamp(int(floor(Vf)), 0, ch); + const int U2 = Core::Maths::clamp(U1 + 1, 0, cw); + const int V2 = Core::Maths::clamp(V1 + 1, 0, ch); + const float s = Uf - U1; + const float t = Vf - V1; + const Core::Maths::Vec4f A = input.GetPixel(U1, V1); + const Core::Maths::Vec4f B = input.GetPixel(U2, V1); + const Core::Maths::Vec4f C = input.GetPixel(U1, V2); + const Core::Maths::Vec4f D = input.GetPixel(U2, V2); + write_cubemap_pixel(face, i, j, A * (1 - s) * (1 - t) + B * s * (1 - t) + C * (1 - s) * t + D * s * t); + } + } + } + return out; + } + Bitmap CrossToCubemap(const Bitmap& input, Core::Memory::TLSFSlab* slab) { const int face_w = input.Width / 3; diff --git a/ZEngine/ZEngine/Rendering/Buffers/Bitmap.h b/ZEngine/ZEngine/Rendering/Buffers/Bitmap.h index bef743059..e1fa00b58 100644 --- a/ZEngine/ZEngine/Rendering/Buffers/Bitmap.h +++ b/ZEngine/ZEngine/Rendering/Buffers/Bitmap.h @@ -59,6 +59,9 @@ namespace ZEngine::Rendering::Buffers namespace BitmapConvert { Bitmap EquirectToCross(const Bitmap& equirect, Core::Memory::TLSFSlab* slab = nullptr); + /// @brief Converts a validated 2:1 equirectangular image into the renderer's canonical cubemap face order. + /// @details This avoids allocating a transient vertical-cross image during HDRI cooking. + Bitmap EquirectToCubemap(const Bitmap& equirect, Core::Memory::TLSFSlab* slab = nullptr); Bitmap CrossToCubemap(const Bitmap& cross, Core::Memory::TLSFSlab* slab = nullptr); } // namespace BitmapConvert diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp index e546f29cd..1ff672ecc 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp @@ -1786,7 +1786,12 @@ namespace ZEngine::Rendering cstring file_ext = std::strrchr(filename, '.'); if (!file_ext) file_ext = ""; - const bool is_environment_map = Helpers::secure_strcmp(file_ext, ".zenvmap") == 0; + const bool is_environment_map = Helpers::secure_strcmp(file_ext, ".zenvmap") == 0; + if (Helpers::secure_strcmp(file_ext, ".hdr") == 0 || Helpers::secure_strcmp(file_ext, ".exr") == 0) + { + ZENGINE_CORE_ERROR("[RRM] Raw HDR environment maps must be imported into a .zenvmap artifact before runtime upload: {}", filename) + return {}; + } TextureSpecification spec{}; @@ -1810,20 +1815,9 @@ namespace ZEngine::Rendering if (!stbi_info(filename, &w, &h, &ch)) return {}; - const bool is_equirectangular = Helpers::secure_strcmp(file_ext, ".hdr") == 0 || Helpers::secure_strcmp(file_ext, ".exr") == 0; - spec.Width = static_cast(w); - spec.Height = static_cast(h); - spec.Format = ImageFormat::R8G8B8A8_SRGB; - - if (is_equirectangular) - { - int face_size = w / 4; - spec.IsCubemap = true; - spec.LayerCount = 6; - spec.Format = ImageFormat::R32G32B32A32_SFLOAT; - spec.Width = static_cast(face_size); - spec.Height = static_cast(face_size); - } + spec.Width = static_cast(w); + spec.Height = static_cast(h); + spec.Format = ImageFormat::R8G8B8A8_SRGB; } if (spec.IsCubemap) @@ -1913,38 +1907,7 @@ namespace ZEngine::Rendering } else { - int width = 0, height = 0, channels = 0; - const float* image_data = stbi_loadf(task->Filename, &width, &height, &channels, STBI_rgb_alpha); - if (!image_data) - { - ZENGINE_CORE_ERROR("Failed to load texture: {}", task->Filename) - } - else - { - const size_t total_pixels = static_cast(width) * static_cast(height); - const size_t float_bytes = total_pixels * STBI_rgb_alpha * sizeof(float); - float* rgba = static_cast(slab->Alloc(float_bytes)); - if (channels == STBI_rgb) - { - stbir_resize_float(image_data, width, height, 0, rgba, width, height, 0, STBI_rgb_alpha); - for (size_t i = 0; i < total_pixels; ++i) - rgba[i * STBI_rgb_alpha + 3] = 255.f; - } - else - { - Helpers::secure_memcpy(rgba, float_bytes, image_data, float_bytes); - } - stbi_image_free(const_cast(image_data)); - - Rendering::Buffers::Bitmap input = Rendering::Buffers::Bitmap::FromData(width, height, 1, STBI_rgb_alpha, Rendering::Buffers::BitmapFormat::Float, Rendering::Buffers::BitmapType::Texture2D, rgba, slab); - slab->Free(rgba); - Rendering::Buffers::Bitmap cross = Rendering::Buffers::BitmapConvert::EquirectToCross(input, slab); - Rendering::Buffers::Bitmap cubemap = Rendering::Buffers::BitmapConvert::CrossToCubemap(cross, slab); - - byte_size = cubemap.BufferSize; - pixels = static_cast(slab->Alloc(byte_size)); - Helpers::secure_memmove(pixels, byte_size, cubemap.Buffer, byte_size); - } + ZENGINE_CORE_ERROR("[RRM] Cubemap uploads require a cooked .zenvmap artifact: {}", task->Filename) } } else diff --git a/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp b/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp index a9c2d7c2c..805fdc0c3 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -262,8 +263,22 @@ namespace ZEngine::Rendering::Renderers void GraphicRenderer::ApplySkyConfig(const Scenes::SkyConfig& sky, const Scenes::SkyCelestialLight& celestial_light, uint64_t revision) { - const EnvironmentLightingBakeSettings bake_settings = Device ? Device->EnvironmentLightingBakeSettings : ResolveEnvironmentLightingQuality(EnvironmentLightingQualityTier::Standard); - if (m_sky_environment.SubmitConfig(sky, revision, bake_settings, celestial_light)) + const EnvironmentLightingBakeSettings bake_settings = Device ? Device->EnvironmentLightingBakeSettings : ResolveEnvironmentLightingQuality(EnvironmentLightingQualityTier::Standard); + uint64_t hdri_source_hash = 0; + bool hdri_artifact_ready = !sky.IsHDRI(); + if (sky.IsHDRI()) + { + if (auto* const asset_manager = ZEngine::Managers::AssetManager::Instance(); asset_manager && asset_manager->Registry) + { + if (const auto* const environment = asset_manager->Registry->FindByUUID(sky.EnvironmentMap)) + { + hdri_source_hash = environment->Meta.SourceHash; + hdri_artifact_ready = environment->State == Core::VFS::AssetState::Loaded && environment->Meta.ArtifactPath[0] != '\0'; + } + } + } + + if (m_sky_environment.SubmitConfig(sky, revision, bake_settings, celestial_light, hdri_source_hash, hdri_artifact_ready)) StartPendingSkyBake(); PollSkyBake(); } @@ -388,11 +403,34 @@ namespace ZEngine::Rendering::Renderers return; } + if (environment->Meta.ArtifactPath[0] == '\0') + { + ZENGINE_CORE_WARN("[SkyEnvironment] Revision {} is using the fallback: HDRI has no completed cooked artifact", request.Revision) + m_sky_environment.CompleteBake(request.Revision, {}, false); + return; + } + + const auto artifact_path = Core::VFS::VFSPath::Parse(environment->Meta.ArtifactPath); + if (artifact_path.Failed()) + { + ZENGINE_CORE_ERROR("[SkyEnvironment] Revision {} is using the fallback: HDRI cooked-artifact path is invalid", request.Revision) + m_sky_environment.CompleteBake(request.Revision, {}, false); + return; + } + char native_path[MAX_FILE_PATH_COUNT] = {}; - environment->Path.ResolveNative(asset_manager->CurrentWorkingSpacePath, native_path, sizeof(native_path)); + artifact_path.Value().ResolveNative(asset_manager->CurrentWorkingSpacePath, native_path, sizeof(native_path)); if (native_path[0] == '\0') { - ZENGINE_CORE_ERROR("[SkyEnvironment] Revision {} is using the fallback: HDRI path cannot be resolved", request.Revision) + ZENGINE_CORE_ERROR("[SkyEnvironment] Revision {} is using the fallback: HDRI cooked-artifact path cannot be resolved", request.Revision) + m_sky_environment.CompleteBake(request.Revision, {}, false); + return; + } + + Importers::AssetCodec::EnvironmentMapFileHeader artifact_header = {}; + if (!Importers::AssetCodec::ReadEnvironmentMapFileHeader(native_path, artifact_header) || !Importers::AssetCodec::DoesEnvironmentMapHeaderMatchSource(artifact_header, request.HDRISourceHash)) + { + ZENGINE_CORE_ERROR("[SkyEnvironment] Revision {} is using the fallback: HDRI cooked artifact is stale, corrupt, or incompatible", request.Revision) m_sky_environment.CompleteBake(request.Revision, {}, false); return; } @@ -437,6 +475,16 @@ namespace ZEngine::Rendering::Renderers return; } + if (!m_sky_environment.IsActiveBakeCurrent()) + { + rrm->ForgetTextureDecode(source_radiance); + m_sky_environment.CompleteBake(revision, source_radiance, false); + DiscardSkyTexture(source_radiance); + ZENGINE_CORE_INFO("[SkyEnvironment] Cancelled stale HDRI revision {} before GPU baking", revision) + StartPendingSkyBake(); + return; + } + const Hardwares::StreamingUploadTicket* const ticket = rrm->FindStreamingUploadTicket(source_radiance); if (!ticket || !ticket->CompletionTimeline) return; @@ -474,7 +522,7 @@ namespace ZEngine::Rendering::Renderers // The stage boundary above is also a cancellation point. Do not spend // more GPU work on a superseded source revision. - if (revision != m_sky_environment.GetLatestRevision()) + if (!m_sky_environment.IsActiveBakeCurrent()) { const Scenes::AtmosphereStaticResources atmosphere = m_sky_environment.GetActiveBakeAtmosphere(); const bool owns_atmosphere = m_sky_environment.ActiveBakeOwnsAtmosphere(); diff --git a/ZEngine/ZEngine/Rendering/Scenes/SkyEnvironment.cpp b/ZEngine/ZEngine/Rendering/Scenes/SkyEnvironment.cpp index f2aace299..2ccb8b8c6 100644 --- a/ZEngine/ZEngine/Rendering/Scenes/SkyEnvironment.cpp +++ b/ZEngine/ZEngine/Rendering/Scenes/SkyEnvironment.cpp @@ -18,16 +18,25 @@ namespace ZEngine::Rendering::Scenes m_state = SkyEnvironmentState::Fallback; } - bool SkyEnvironment::SubmitConfig(const SkyConfig& config, uint64_t revision, const EnvironmentLightingBakeSettings& bake_settings, const SkyCelestialLight& celestial_light) + bool SkyEnvironment::SubmitConfig(const SkyConfig& config, uint64_t revision, const EnvironmentLightingBakeSettings& bake_settings, const SkyCelestialLight& celestial_light, uint64_t hdri_source_hash, bool hdri_artifact_ready) { - if (revision == 0 || revision <= m_latest_revision) + if (revision == 0 || revision < m_latest_revision) return false; const bool has_required_sun = !config.IsAtmosphere() || celestial_light.IsAvailable; - const bool inputs_valid = config.IsValid() && celestial_light.IsValid() && has_required_sun; SkyConfig sanitized = config; const EnvironmentLightingBakeSettings resolved_bake_settings = bake_settings.IsValid() ? bake_settings : ResolveEnvironmentLightingQuality(EnvironmentLightingQualityTier::Standard); sanitized.Sanitize(); + const bool is_hdri = sanitized.IsHDRI(); + const bool source_is_ready = !is_hdri || hdri_artifact_ready; + const bool inputs_valid = config.IsValid() && celestial_light.IsValid() && has_required_sun && source_is_ready; + + // A normal scene revision remains immutable. The exception is an HDRI + // source transition: imports keep the scene's UUID and revision stable, + // but a new source hash or cooked-artifact readiness must replace any + // pending/active work for the prior source. + if (revision == m_latest_revision && (!is_hdri || (m_bake_config.IsHDRI() && m_bake_config.EnvironmentMap == sanitized.EnvironmentMap && m_bake_hdri_source_hash == hdri_source_hash && m_bake_hdri_artifact_ready == hdri_artifact_ready))) + return false; m_presentation_config = sanitized; m_presentation_celestial_light = celestial_light.IsValid() ? celestial_light : SkyCelestialLight{}; @@ -42,18 +51,20 @@ namespace ZEngine::Rendering::Scenes if (!published.IsFallback) published.Retired = true; - m_published_slot = 0; - m_pending_request = {}; - m_has_pending_request = false; - m_bake_config.Mode = static_cast(UINT8_MAX); - m_bake_celestial_light = {}; - m_bake_inputs_valid = false; - m_latest_bake_revision = revision; - m_state = SkyEnvironmentState::Fallback; + m_published_slot = 0; + m_pending_request = {}; + m_has_pending_request = false; + m_bake_config.Mode = static_cast(UINT8_MAX); + m_bake_celestial_light = {}; + m_bake_inputs_valid = false; + m_bake_hdri_source_hash = 0; + m_bake_hdri_artifact_ready = true; + m_latest_bake_revision = revision; + m_state = SkyEnvironmentState::Fallback; return true; } - if (inputs_valid && m_bake_inputs_valid && HasEquivalentBakeInputs(m_bake_config, m_bake_celestial_light, sanitized, celestial_light) && m_bake_settings.Matches(resolved_bake_settings)) + if (inputs_valid && m_bake_inputs_valid && HasEquivalentBakeInputs(m_bake_config, m_bake_celestial_light, m_bake_hdri_source_hash, m_bake_hdri_artifact_ready, sanitized, celestial_light, hdri_source_hash, hdri_artifact_ready) && m_bake_settings.Matches(resolved_bake_settings)) { // The source radiance remains valid. Keep editor-facing presentation // changes (tint, intensity, and yaw) off the bake path. @@ -75,17 +86,21 @@ namespace ZEngine::Rendering::Scenes return true; } - m_bake_config = sanitized; - m_bake_celestial_light = celestial_light; - m_bake_inputs_valid = inputs_valid; - m_bake_settings = resolved_bake_settings; - m_pending_request.Config = sanitized; - m_pending_request.CelestialLight = celestial_light; - m_pending_request.BakeSettings = resolved_bake_settings; - m_pending_request.Revision = revision; - m_pending_request.BakeInputsValid = inputs_valid; - m_latest_bake_revision = revision; - m_has_pending_request = true; + m_bake_config = sanitized; + m_bake_celestial_light = celestial_light; + m_bake_hdri_source_hash = is_hdri ? hdri_source_hash : 0; + m_bake_hdri_artifact_ready = source_is_ready; + m_bake_inputs_valid = inputs_valid; + m_bake_settings = resolved_bake_settings; + m_pending_request.Config = sanitized; + m_pending_request.CelestialLight = celestial_light; + m_pending_request.BakeSettings = resolved_bake_settings; + m_pending_request.HDRISourceHash = is_hdri ? hdri_source_hash : 0; + m_pending_request.Revision = revision; + m_pending_request.HDRIArtifactReady = source_is_ready; + m_pending_request.BakeInputsValid = inputs_valid; + m_latest_bake_revision = revision; + m_has_pending_request = true; return true; } @@ -212,28 +227,34 @@ namespace ZEngine::Rendering::Scenes return m_has_active_bake && m_active_bake_stage == SkyEnvironmentBakeStage::ReadyToPublish && !m_active_stage_submitted; } + bool SkyEnvironment::IsActiveBakeCurrent() const + { + return m_has_active_bake && m_active_bake.Revision == m_latest_bake_revision && m_active_bake.BakeSettings.Matches(m_bake_settings) && HasEquivalentBakeInputs(m_active_bake.Config, m_active_bake.CelestialLight, m_active_bake.HDRISourceHash, m_active_bake.HDRIArtifactReady, m_bake_config, m_bake_celestial_light, m_bake_hdri_source_hash, m_bake_hdri_artifact_ready); + } + SkyEnvironmentBakeResult SkyEnvironment::CompleteBake(uint64_t revision, Textures::TextureHandle source_radiance, bool success, const EnvironmentLightingResources& lighting, const AtmosphereStaticResources& atmosphere) { if (!m_has_active_bake || m_active_bake.Revision != revision) return SkyEnvironmentBakeResult::Ignored; - const bool requires_atmosphere = m_active_bake.Config.IsAtmosphere(); - const SkyCelestialLight completed_celestial = m_active_bake.CelestialLight; - const Textures::TextureHandle completed_source = source_radiance.Valid() ? source_radiance : m_active_bake_source; - const EnvironmentLightingResources completed_lighting = lighting.Valid() ? lighting : m_active_bake_lighting.Valid() ? m_active_bake_lighting : m_fallback_lighting; - const AtmosphereStaticResources completed_atmosphere = atmosphere.Valid() ? atmosphere : m_active_bake_atmosphere; - m_active_bake = {}; - m_active_bake_atmosphere = {}; - m_active_bake_owns_atmosphere = false; - m_active_bake_source = {}; - m_active_bake_lighting = {}; - m_active_bake_stage = SkyEnvironmentBakeStage::AwaitingSource; - m_active_stage_timeline = 0; - m_active_stage_submitted = false; - m_active_stage_recorded = false; - m_has_active_bake = false; - - if (revision != m_latest_bake_revision) + const bool active_bake_is_current = IsActiveBakeCurrent(); + const bool requires_atmosphere = m_active_bake.Config.IsAtmosphere(); + const SkyCelestialLight completed_celestial = m_active_bake.CelestialLight; + const Textures::TextureHandle completed_source = source_radiance.Valid() ? source_radiance : m_active_bake_source; + const EnvironmentLightingResources completed_lighting = lighting.Valid() ? lighting : m_active_bake_lighting.Valid() ? m_active_bake_lighting : m_fallback_lighting; + const AtmosphereStaticResources completed_atmosphere = atmosphere.Valid() ? atmosphere : m_active_bake_atmosphere; + m_active_bake = {}; + m_active_bake_atmosphere = {}; + m_active_bake_owns_atmosphere = false; + m_active_bake_source = {}; + m_active_bake_lighting = {}; + m_active_bake_stage = SkyEnvironmentBakeStage::AwaitingSource; + m_active_stage_timeline = 0; + m_active_stage_submitted = false; + m_active_stage_recorded = false; + m_has_active_bake = false; + + if (!active_bake_is_current) return SkyEnvironmentBakeResult::Discarded; if (!success || !completed_source.Valid() || (requires_atmosphere && !completed_atmosphere.Valid())) @@ -423,12 +444,12 @@ namespace ZEngine::Rendering::Scenes first.MieAnisotropy == second.MieAnisotropy && equal3(first.OzoneAbsorptionPerKilometer, second.OzoneAbsorptionPerKilometer) && first.OzoneCenterKilometers == second.OzoneCenterKilometers && first.OzoneThicknessKilometers == second.OzoneThicknessKilometers; } - bool SkyEnvironment::HasEquivalentBakeInputs(const SkyConfig& left, const SkyCelestialLight& left_celestial_light, const SkyConfig& right, const SkyCelestialLight& right_celestial_light) + bool SkyEnvironment::HasEquivalentBakeInputs(const SkyConfig& left, const SkyCelestialLight& left_celestial_light, uint64_t left_hdri_source_hash, bool left_hdri_artifact_ready, const SkyConfig& right, const SkyCelestialLight& right_celestial_light, uint64_t right_hdri_source_hash, bool right_hdri_artifact_ready) { if (left.Mode != right.Mode) return false; if (left.IsHDRI()) - return left.EnvironmentMap == right.EnvironmentMap; + return left.EnvironmentMap == right.EnvironmentMap && left_hdri_source_hash == right_hdri_source_hash && left_hdri_artifact_ready == right_hdri_artifact_ready; if (left.IsSkySphere()) return true; diff --git a/ZEngine/ZEngine/Rendering/Scenes/SkyEnvironment.h b/ZEngine/ZEngine/Rendering/Scenes/SkyEnvironment.h index e07a1d37f..808d2ce3e 100644 --- a/ZEngine/ZEngine/Rendering/Scenes/SkyEnvironment.h +++ b/ZEngine/ZEngine/Rendering/Scenes/SkyEnvironment.h @@ -70,12 +70,18 @@ namespace ZEngine::Rendering::Scenes /// @brief Immutable bake input claimed by the render thread. struct SkyEnvironmentBakeRequest { - SkyConfig Config = {}; - SkyCelestialLight CelestialLight = {}; - EnvironmentLightingBakeSettings BakeSettings = {}; - uint64_t Revision = 0; + SkyConfig Config = {}; + SkyCelestialLight CelestialLight = {}; + EnvironmentLightingBakeSettings BakeSettings = {}; + /// @brief Registry snapshot for the selected HDRI source. This keeps + /// source reimports distinct even though scene data stores only a UUID. + uint64_t HDRISourceHash = 0; + uint64_t Revision = 0; + /// @brief False until the registry has a completed cooked artifact for + /// the selected HDRI. Ignored by atmosphere and SkySphere modes. + bool HDRIArtifactReady = true; /// @brief True only when this request has every valid input needed to bake. - bool BakeInputsValid = true; + bool BakeInputsValid = true; }; /// @brief Result of a revision-tagged environment bake completion. @@ -103,7 +109,7 @@ namespace ZEngine::Rendering::Scenes /// @brief Coalesces an immutable config revision while preserving its identity. /// @return False if the revision is stale or already observed. - bool SubmitConfig(const SkyConfig& config, uint64_t revision, const EnvironmentLightingBakeSettings& bake_settings = {}, const SkyCelestialLight& celestial_light = {}); + bool SubmitConfig(const SkyConfig& config, uint64_t revision, const EnvironmentLightingBakeSettings& bake_settings = {}, const SkyCelestialLight& celestial_light = {}, uint64_t hdri_source_hash = 0, bool hdri_artifact_ready = true); /// @brief Claims the newest revision when no other bake is in flight. bool TakeBakeRequest(SkyEnvironmentBakeRequest& out_request); @@ -125,6 +131,9 @@ namespace ZEngine::Rendering::Scenes /// @brief Advances one completed stage, exposing a cancellation point before the next one. bool AdvanceCompletedGpuBakeStage(uint64_t completed_timeline_value); [[nodiscard]] bool IsGpuBakeReadyToPublish() const; + /// @brief Returns false once a newer config, quality setting, or HDRI + /// source snapshot supersedes the active request. + [[nodiscard]] bool IsActiveBakeCurrent() const; /// @brief Completes a bake, publishing it only if its revision is still current. SkyEnvironmentBakeResult CompleteBake(uint64_t revision, Textures::TextureHandle source_radiance, bool success, const EnvironmentLightingResources& lighting = {}, const AtmosphereStaticResources& atmosphere = {}); @@ -162,7 +171,7 @@ namespace ZEngine::Rendering::Scenes private: [[nodiscard]] static bool HasEquivalentAtmosphereStaticInputs(const SkyConfig& left, const SkyConfig& right); - [[nodiscard]] static bool HasEquivalentBakeInputs(const SkyConfig& left, const SkyCelestialLight& left_celestial_light, const SkyConfig& right, const SkyCelestialLight& right_celestial_light); + [[nodiscard]] static bool HasEquivalentBakeInputs(const SkyConfig& left, const SkyCelestialLight& left_celestial_light, uint64_t left_hdri_source_hash, bool left_hdri_artifact_ready, const SkyConfig& right, const SkyCelestialLight& right_celestial_light, uint64_t right_hdri_source_hash, bool right_hdri_artifact_ready); [[nodiscard]] bool IsAtmosphereShared(uint32_t excluded_snapshot_slot, const AtmosphereStaticResources& atmosphere) const; void ReleaseNextFramePin(uint64_t timeline_value); int32_t FindFreeSnapshotSlot() const; @@ -179,6 +188,7 @@ namespace ZEngine::Rendering::Scenes SkyCelestialLight m_presentation_celestial_light = {}; SkyConfig m_bake_config = {}; SkyCelestialLight m_bake_celestial_light = {}; + uint64_t m_bake_hdri_source_hash = 0; uint16_t m_frame_pin_slots[MaxPendingFramePins] = {}; uint32_t m_published_slot = 0; uint32_t m_frame_pin_head = 0; @@ -191,6 +201,7 @@ namespace ZEngine::Rendering::Scenes bool m_has_active_bake = false; bool m_active_bake_owns_atmosphere = false; bool m_bake_inputs_valid = false; + bool m_bake_hdri_artifact_ready = true; bool m_active_stage_submitted = false; bool m_active_stage_recorded = false; SkyEnvironmentState m_state = SkyEnvironmentState::Fallback; diff --git a/ZEngine/docs/future-plan/sky-rendering.md b/ZEngine/docs/future-plan/sky-rendering.md index 2ff3ef047..50dbff211 100644 --- a/ZEngine/docs/future-plan/sky-rendering.md +++ b/ZEngine/docs/future-plan/sky-rendering.md @@ -402,9 +402,9 @@ Version 1 supports HDR input and extends the current importer that cooks it into EXR is not exposed as supported input until an EXR-capable decoder is integrated and covered by import tests. The current stb_image-based loader must not advertise EXR merely because the importer extension filter accepts it; it should reject unsupported input at import time with a useful error. -Cooked metadata includes source hash, import quality tier, cubemap orientation, colour interpretation, exposure calibration, face size, mip availability, and importer version. The runtime can then invalidate only stale cooked data. +The current `.zenvmap` artifact is version 2. Its fixed header records source hash, importer version, RGBA32F payload contract, linear-scene colour, renderer-canonical cubemap orientation, exposure, face dimensions/layers, and the full GPU-generated mip policy. The runtime validates that complete contract and the exact payload length before it allocates or uploads; a header whose source hash does not match the registry is stale and is never used. -Raw RGBA32F equirectangular images are not retained in resident VRAM merely for backdrop rendering. The cooked cubemap uses half precision or a platform-appropriate HDR compression/streaming format, includes required mips, and is streamed according to an explicit project quality setting. Heap pressure may request eviction or a lower already-cooked tier; it must not silently downscale authored assets at runtime. +Raw HDR equirectangular images are never decoded on the render thread or retained in resident VRAM for backdrop rendering. Version 2 uses an RGBA32F base-level cubemap payload (up to 1024 pixels per face) and generates the required full mip chain on the GPU before IBL convolution. A future half-precision or compressed artifact must receive a new artifact version and preserve the same validation and orientation contract. Heap pressure may request eviction or a lower already-cooked tier; it must not silently downscale authored assets at runtime. Cooked environment artifacts are derived cache data, not source content-browser items. The source asset remains the user-visible item and is the only identity serialized by the scene. Cache eviction or regeneration therefore never changes content-browser structure or source-control state. @@ -414,9 +414,9 @@ HDRI rotation/orientation, tint, and lighting intensity are common source-radian ### 8.1 Import validation and colour contract -The importer validates an equirectangular 2:1 source aspect ratio, finite pixel values, supported dimensions, and a bounded decoded working set before allocating conversion buffers. HDR source pixels are interpreted as linear scene radiance, never as sRGB. Invalid NaN/infinite values fail import; negative radiance values are rejected or clamped according to a documented import policy with a diagnostic. +The importer currently accepts `.hdr` only; `.exr` remains unsupported until an EXR-capable decoder is integrated. It validates an equirectangular 2:1 source aspect ratio, finite non-negative RGBA pixels, width divisible by four, and a maximum 1024-pixel cubemap face before allocating conversion buffers. HDR source pixels are interpreted as linear scene radiance, never as sRGB. Invalid NaN/infinite or negative radiance fails import with a diagnostic. -The cooker writes atomically and preserves the prior valid artifact if recooking fails. It generates or records a complete mip chain, validates the six-face orientation with a canonical direction test, and stores the source/import hash used for stale-artifact detection. The runtime uses only completed artifacts and never samples a partially written cache file. +The cooker writes atomically and preserves the prior valid artifact if recooking fails. It converts directly into the established six-face canonical order (covered against the prior vertical-cross conversion), records the complete GPU-generated mip policy, and stores the source/import hash used for stale-artifact detection. The runtime uses only completed artifacts and never samples a partially written cache file. Registry source hash and artifact readiness are part of the immutable HDRI bake key, so a reimport with the same scene UUID invalidates pending or active work while an unchanged unavailable artifact produces no retry loop. --- diff --git a/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp b/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp new file mode 100644 index 000000000..492b9b7f5 --- /dev/null +++ b/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp @@ -0,0 +1,155 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ZEngine::Importers; +using namespace ZEngine::Importers::AssetCodec; +using namespace ZEngine::Rendering::Buffers; + +namespace +{ + EnvironmentMapFileHeader MakeValidHeader(uint32_t face_size = 2, uint64_t source_hash = 0xAABBCCDDULL) + { + return { + .MagicNumber = ZENVMAP_MAGIC, + .Version = ENVIRONMENT_MAP_FILE_VERSION, + .HeaderByteSize = sizeof(EnvironmentMapFileHeader), + .ImporterVersion = ENVIRONMENT_MAP_IMPORTER_VERSION, + .SourceHash = source_hash, + .FaceWidth = face_size, + .FaceHeight = face_size, + .Channel = 4, + .LayerCount = 6, + .MipCount = GetEnvironmentMapFullMipCount(face_size), + .ColorSpace = static_cast(EnvironmentMapColorSpace::LinearScene), + .Orientation = static_cast(EnvironmentMapOrientation::RendererCanonical), + .MipPolicy = static_cast(EnvironmentMapMipPolicy::GenerateOnGpu), + .Exposure = 1.0f, + .BufferByteSize = static_cast(face_size) * face_size * 6 * 4 * sizeof(float), + }; + } +} // namespace + +TEST(EnvironmentMapCookingTest, ImporterClaimsOnlySupportedHdrSources) +{ + EnvironmentMapImporter importer = {}; + + EXPECT_TRUE(importer.CanImport("hdr")); + EXPECT_FALSE(importer.CanImport("exr")); + EXPECT_FALSE(importer.CanImport("png")); + EXPECT_FALSE(importer.CanImport(nullptr)); +} + +TEST(EnvironmentMapCookingTest, SourceValidationRejectsMalformedAndOversizedInputs) +{ + std::array pixels = {}; + pixels.fill(1.0f); + + EXPECT_TRUE(EnvironmentMapImporter::IsSupportedEquirectangularSource(4, 2, pixels.data())); + EXPECT_FALSE(EnvironmentMapImporter::IsSupportedEquirectangularSource(6, 2, pixels.data())); + EXPECT_FALSE(EnvironmentMapImporter::IsSupportedEquirectangularSource(4, 2, nullptr)); + + pixels[0] = -0.5f; + EXPECT_FALSE(EnvironmentMapImporter::IsSupportedEquirectangularSource(4, 2, pixels.data())); + pixels[0] = std::numeric_limits::quiet_NaN(); + EXPECT_FALSE(EnvironmentMapImporter::IsSupportedEquirectangularSource(4, 2, pixels.data())); + + float placeholder = 1.0f; + EXPECT_FALSE(EnvironmentMapImporter::IsSupportedEquirectangularSource(static_cast(AssetCodec::ENVIRONMENT_MAP_MAX_FACE_SIZE * 4 + 4), static_cast(AssetCodec::ENVIRONMENT_MAP_MAX_FACE_SIZE * 2 + 2), &placeholder)); +} + +TEST(EnvironmentMapCookingTest, ArtifactPathIsStableAndCacheOnly) +{ + const uuids::uuid asset_uuid = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440000").value(); + char artifact_path[256] = {}; + + ASSERT_TRUE(EnvironmentMapImporter::BuildArtifactPath(asset_uuid, artifact_path, sizeof(artifact_path))); + EXPECT_STREQ(artifact_path, "/_cache/envmaps/550e8400-e29b-41d4-a716-446655440000.zenvmap"); + EXPECT_FALSE(EnvironmentMapImporter::BuildArtifactPath({}, artifact_path, sizeof(artifact_path))); +} + +TEST(EnvironmentMapCookingTest, HeaderValidationRejectsWrongContractsAndStaleSources) +{ + EnvironmentMapFileHeader header = MakeValidHeader(); + ASSERT_TRUE(IsEnvironmentMapFileHeaderValid(header)); + EXPECT_TRUE(DoesEnvironmentMapHeaderMatchSource(header, 0xAABBCCDDULL)); + EXPECT_FALSE(DoesEnvironmentMapHeaderMatchSource(header, 0xDDCCBBAAULL)); + + header.Orientation = 0; + EXPECT_FALSE(IsEnvironmentMapFileHeaderValid(header)); + header = MakeValidHeader(); + ++header.BufferByteSize; + EXPECT_FALSE(IsEnvironmentMapFileHeaderValid(header)); +} + +TEST(EnvironmentMapCookingTest, DeserializerRejectsTruncatedCookedArtifact) +{ + const std::filesystem::path artifact_path = std::filesystem::temp_directory_path() / "zengine_environment_map_cooking_test.zenvmap"; + const EnvironmentMapFileHeader header = MakeValidHeader(); + const std::vector payload(static_cast(header.BufferByteSize) / sizeof(float), 0.25f); + + { + std::ofstream output(artifact_path, std::ios::binary | std::ios::trunc); + ASSERT_TRUE(output.is_open()); + output.write(reinterpret_cast(&header), sizeof(header)); + output.write(reinterpret_cast(payload.data()), static_cast(header.BufferByteSize)); + } + + Bitmap decoded = {}; + ASSERT_TRUE(DeserializeEnvironmentMapFile(artifact_path.c_str(), decoded)); + EXPECT_EQ(decoded.Type, BitmapType::CubeMap); + EXPECT_EQ(decoded.Width, 2); + EXPECT_EQ(decoded.Layers, 6); + + { + std::ofstream output(artifact_path, std::ios::binary | std::ios::trunc); + ASSERT_TRUE(output.is_open()); + output.write(reinterpret_cast(&header), sizeof(header)); + } + + Bitmap corrupt = {}; + EXPECT_FALSE(DeserializeEnvironmentMapFile(artifact_path.c_str(), corrupt)); + std::error_code error; + std::filesystem::remove(artifact_path, error); +} + +TEST(EnvironmentMapCookingTest, DirectConversionPreservesTheCanonicalFaceOrientation) +{ + constexpr int width = 8; + constexpr int height = 4; + std::array pixels = {}; + for (int y = 0; y < height; ++y) + { + for (int x = 0; x < width; ++x) + { + const size_t index = static_cast(y * width + x) * 4; + pixels[index + 0] = static_cast(x + y * width); + pixels[index + 1] = static_cast(x); + pixels[index + 2] = static_cast(y); + pixels[index + 3] = 1.0f; + } + } + + Bitmap equirectangular = Bitmap::FromData(width, height, 1, 4, BitmapFormat::Float, BitmapType::Texture2D, pixels.data()); + Bitmap cross = BitmapConvert::EquirectToCross(equirectangular); + Bitmap expected = BitmapConvert::CrossToCubemap(cross); + Bitmap actual = BitmapConvert::EquirectToCubemap(equirectangular); + + ASSERT_EQ(actual.Type, BitmapType::CubeMap); + ASSERT_EQ(actual.BufferSize, expected.BufferSize); + const auto* const expected_pixels = reinterpret_cast(expected.Buffer); + const auto* const actual_pixels = reinterpret_cast(actual.Buffer); + for (size_t index = 0; index < actual.BufferSize / sizeof(float); ++index) + EXPECT_FLOAT_EQ(actual_pixels[index], expected_pixels[index]) << "component " << index; +} diff --git a/ZEngine/tests/Rendering/SkyEnvironment_test.cpp b/ZEngine/tests/Rendering/SkyEnvironment_test.cpp index 2dcc50142..d187573de 100644 --- a/ZEngine/tests/Rendering/SkyEnvironment_test.cpp +++ b/ZEngine/tests/Rendering/SkyEnvironment_test.cpp @@ -190,6 +190,44 @@ TEST(SkyEnvironmentTest, PresentationChangesKeepAnInFlightBakeCurrent) EXPECT_FLOAT_EQ(environment.GetPublishedSnapshot()->Config.EnvironmentIntensity, 2.0f); } +TEST(SkyEnvironmentTest, HdrSourceReloadAtTheSameSceneRevisionDiscardsStaleWork) +{ + SkyEnvironment environment = {}; + environment.Initialize(Texture(1), Lighting(10)); + + SkyConfig config = HDRIConfig(); + config.EnvironmentMap = uuids::uuid::from_string("550e8400-e29b-41d4-a716-446655440000").value(); + + SkyEnvironmentBakeRequest request = {}; + ASSERT_TRUE(environment.SubmitConfig(config, 1, {}, {}, 0x1111ULL, true)); + ASSERT_TRUE(environment.TakeBakeRequest(request)); + ASSERT_TRUE(environment.AttachBakeResource(1, Texture(2))); + EXPECT_TRUE(environment.IsActiveBakeCurrent()); + + // The source enters the stale/importing state before its new artifact is + // ready. Its scene UUID and scene revision intentionally remain unchanged. + ASSERT_TRUE(environment.SubmitConfig(config, 1, {}, {}, 0x2222ULL, false)); + EXPECT_FALSE(environment.IsActiveBakeCurrent()); + EXPECT_EQ(environment.CompleteBake(1, Texture(2), true), SkyEnvironmentBakeResult::Discarded); + + ASSERT_TRUE(environment.TakeBakeRequest(request)); + EXPECT_EQ(request.Revision, 1u); + EXPECT_EQ(request.HDRISourceHash, 0x2222ULL); + EXPECT_FALSE(request.HDRIArtifactReady); + EXPECT_FALSE(request.BakeInputsValid); + EXPECT_EQ(environment.CompleteBake(1, {}, false), SkyEnvironmentBakeResult::Failed); + + // An unchanged unavailable artifact does not cause per-frame retry spam. + EXPECT_FALSE(environment.SubmitConfig(config, 1, {}, {}, 0x2222ULL, false)); + + // Import completion uses the same UUID/hash but marks the cooked artifact + // ready, which schedules exactly one fresh bake. + ASSERT_TRUE(environment.SubmitConfig(config, 1, {}, {}, 0x2222ULL, true)); + ASSERT_TRUE(environment.TakeBakeRequest(request)); + EXPECT_TRUE(request.HDRIArtifactReady); + EXPECT_TRUE(request.BakeInputsValid); +} + TEST(SkyEnvironmentTest, SkySphereCancelsStaleBakeAndKeepsTheFallback) { SkyEnvironment environment = {}; diff --git a/ZEngine/tests/Rendering/TextureImporterTest.cpp b/ZEngine/tests/Rendering/TextureImporterTest.cpp index ec9b9a76b..bcc494486 100644 --- a/ZEngine/tests/Rendering/TextureImporterTest.cpp +++ b/ZEngine/tests/Rendering/TextureImporterTest.cpp @@ -16,9 +16,9 @@ TEST(TextureImporterTest, CanImportClaimsAllEightRasterExtensions) TEST(TextureImporterTest, CanImportDoesNotClaimEnvironmentMapOrContainerFormats) { TextureImporter importer; - // hdr/exr stay EnvironmentMapImporter's domain; ktx/ktx2 are recognized by - // AssetRegistry::InferTypeFromExtension but stb_image cannot decode them — neither - // should be claimed here, or ImportCoordinator's first-match routing gets ambiguous. + // HDR is claimed by EnvironmentMapImporter; EXR and KTX variants remain + // recognized by AssetRegistry::InferTypeFromExtension but unsupported until + // their dedicated decoders are introduced. None belong to this importer. const char* unclaimed[] = {"hdr", "exr", "ktx", "ktx2", "zenvmap", "glb", "fbx", "obj"}; for (const char* ext : unclaimed) EXPECT_FALSE(importer.CanImport(ext)) << ext; From fbb4777c3030e5f5a271ebc130d9ef3daf8e7201 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Wed, 16 Sep 2026 13:32:49 +0900 Subject: [PATCH 2/4] feat(import): support EXR environment maps --- .../Importers/EnvironmentMapImporter.cpp | 41 ++++++++++++--- .../Importers/EnvironmentMapImporter.h | 2 +- ZEngine/docs/future-plan/sky-rendering.md | 4 +- .../Rendering/EnvironmentMapCooking_test.cpp | 51 ++++++++++++++++++- .../tests/Rendering/TextureImporterTest.cpp | 4 +- dependencies.cmake | 19 +++++++ 6 files changed, 106 insertions(+), 15 deletions(-) diff --git a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp index e697df1c3..6e86e8d34 100644 --- a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp +++ b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp @@ -9,9 +9,11 @@ #include #include #include +#include // stb_image implementation is defined once in RenderResourceManager.cpp. #include +#include using namespace ZEngine::Rendering::Buffers; @@ -38,9 +40,7 @@ namespace ZEngine::Importers { if (!extension) return false; - // stb_image does not decode EXR. Do not advertise it until an EXR-capable - // importer is implemented and covered by the same artifact contract. - return Helpers::secure_strcmp(extension, "hdr") == 0; + return Helpers::secure_strcmp(extension, "hdr") == 0 || Helpers::secure_strcmp(extension, "exr") == 0; } bool EnvironmentMapImporter::IsSupportedEquirectangularSource(int width, int height, const float* rgba_pixels) @@ -78,24 +78,49 @@ namespace ZEngine::Importers else path.ToNative(native, sizeof(native)); - int width = 0, height = 0, channel = 0; - const float* image_data = stbi_loadf(native, &width, &height, &channel, STBI_rgb_alpha); + int width = 0, height = 0, channel = 0; + float* image_data = nullptr; + const bool is_exr = path.Extension().Equals(".exr"); + if (is_exr) + { + const char* error_message = nullptr; + if (LoadEXR(&image_data, &width, &height, native, &error_message) != TINYEXR_SUCCESS) + { + ZENGINE_CORE_ERROR("EnvironmentMapImporter: failed to load EXR '{}': {}", native, error_message ? error_message : "unknown decoder error") + if (error_message) + FreeEXRErrorMessage(error_message); + if (image_data) + std::free(image_data); + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); + } + channel = STBI_rgb_alpha; + } + else + { + image_data = const_cast(stbi_loadf(native, &width, &height, &channel, STBI_rgb_alpha)); + } if (!image_data) { - ZENGINE_CORE_ERROR("EnvironmentMapImporter: failed to load '{}': {}", native, stbi_failure_reason()) + ZENGINE_CORE_ERROR("EnvironmentMapImporter: failed to load {} '{}': {}", is_exr ? "EXR" : "HDR", native, is_exr ? "decoder returned no pixels" : stbi_failure_reason()) return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); } if (!IsSupportedEquirectangularSource(width, height, image_data)) { - stbi_image_free(const_cast(image_data)); + if (is_exr) + std::free(image_data); + else + stbi_image_free(image_data); ZENGINE_CORE_ERROR("EnvironmentMapImporter: '{}' must be a finite, non-negative 2:1 HDR equirectangular image with a face size no larger than {}", native, AssetCodec::ENVIRONMENT_MAP_MAX_FACE_SIZE) return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::InvalidPath); } Core::Memory::TLSFSlab* slab = Helpers::GetWorkerSlab(); Bitmap equirect = Bitmap::FromData(width, height, 1, STBI_rgb_alpha, BitmapFormat::Float, BitmapType::Texture2D, image_data); - stbi_image_free(const_cast(image_data)); + if (is_exr) + std::free(image_data); + else + stbi_image_free(image_data); Bitmap cubemap = BitmapConvert::EquirectToCubemap(equirect, slab); diff --git a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.h b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.h index d8e473293..7fc2378df 100644 --- a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.h +++ b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.h @@ -4,7 +4,7 @@ namespace ZEngine::Importers { - // Imports .hdr equirectangular images, converts them to cubemaps, + // Imports .hdr and standard .exr equirectangular images, converts them to cubemaps, // and writes a .zenvmap cooked artifact. Registered with ImportCoordinator // so the standard Enqueue/Tick pipeline handles HDRI assets off the render thread. class EnvironmentMapImporter : public IAssetImporter diff --git a/ZEngine/docs/future-plan/sky-rendering.md b/ZEngine/docs/future-plan/sky-rendering.md index 50dbff211..b6e0ff438 100644 --- a/ZEngine/docs/future-plan/sky-rendering.md +++ b/ZEngine/docs/future-plan/sky-rendering.md @@ -400,7 +400,7 @@ authored .hdr Version 1 supports HDR input and extends the current importer that cooks it into a cubemap cache artifact. It must not also introduce a second runtime raw-equirectangular conversion pipeline. A future GPU-based cooker can replace the conversion implementation behind the same asset contract. -EXR is not exposed as supported input until an EXR-capable decoder is integrated and covered by import tests. The current stb_image-based loader must not advertise EXR merely because the importer extension filter accepts it; it should reject unsupported input at import time with a useful error. +Radiance `.hdr` and flat, single-part `.exr` input are supported. HDR uses stb_image and EXR uses TinyEXR; both decode to the same linear RGBA32F conversion path. Deep, multipart, and other EXR workflows that cannot be represented as one flat image are intentionally not accepted as environment sources. The current `.zenvmap` artifact is version 2. Its fixed header records source hash, importer version, RGBA32F payload contract, linear-scene colour, renderer-canonical cubemap orientation, exposure, face dimensions/layers, and the full GPU-generated mip policy. The runtime validates that complete contract and the exact payload length before it allocates or uploads; a header whose source hash does not match the registry is stale and is never used. @@ -414,7 +414,7 @@ HDRI rotation/orientation, tint, and lighting intensity are common source-radian ### 8.1 Import validation and colour contract -The importer currently accepts `.hdr` only; `.exr` remains unsupported until an EXR-capable decoder is integrated. It validates an equirectangular 2:1 source aspect ratio, finite non-negative RGBA pixels, width divisible by four, and a maximum 1024-pixel cubemap face before allocating conversion buffers. HDR source pixels are interpreted as linear scene radiance, never as sRGB. Invalid NaN/infinite or negative radiance fails import with a diagnostic. +The importer accepts Radiance `.hdr` and flat, single-part `.exr`. It validates an equirectangular 2:1 source aspect ratio, finite non-negative RGBA pixels, width divisible by four, and a maximum 1024-pixel cubemap face before allocating conversion buffers. HDR source pixels are interpreted as linear scene radiance, never as sRGB. Invalid NaN/infinite or negative radiance fails import with a diagnostic. The cooker writes atomically and preserves the prior valid artifact if recooking fails. It converts directly into the established six-face canonical order (covered against the prior vertical-cross conversion), records the complete GPU-generated mip policy, and stores the source/import hash used for stale-artifact detection. The runtime uses only completed artifacts and never samples a partially written cache file. Registry source hash and artifact readiness are part of the immutable HDRI bake key, so a reimport with the same scene UUID invalidates pending or active work while an unchanged unavailable artifact produces no retry loop. diff --git a/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp b/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp index 492b9b7f5..cb45b82a6 100644 --- a/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp +++ b/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -41,16 +42,62 @@ namespace } } // namespace -TEST(EnvironmentMapCookingTest, ImporterClaimsOnlySupportedHdrSources) +TEST(EnvironmentMapCookingTest, ImporterClaimsSupportedHdrAndExrSources) { EnvironmentMapImporter importer = {}; EXPECT_TRUE(importer.CanImport("hdr")); - EXPECT_FALSE(importer.CanImport("exr")); + EXPECT_TRUE(importer.CanImport("exr")); EXPECT_FALSE(importer.CanImport("png")); EXPECT_FALSE(importer.CanImport(nullptr)); } +TEST(EnvironmentMapCookingTest, TinyExrDecodesAFloatEquirectangularSource) +{ + constexpr int width = 4; + constexpr int height = 2; + const std::filesystem::path source_path = std::filesystem::temp_directory_path() / "zengine_environment_map_cooking_test.exr"; + std::array source = {}; + for (size_t index = 0; index < source.size(); ++index) + source[index] = static_cast(index) * 0.25f; + + const char* error_message = nullptr; + const int write_result = SaveEXR(source.data(), width, height, 4, 0, source_path.c_str(), &error_message); + if (write_result != TINYEXR_SUCCESS) + { + ADD_FAILURE() << "TinyEXR failed to write the test source: " << (error_message ? error_message : "unknown error"); + if (error_message) + FreeEXRErrorMessage(error_message); + return; + } + + float* decoded = nullptr; + int decoded_width = 0; + int decoded_height = 0; + error_message = nullptr; + const int read_result = LoadEXR(&decoded, &decoded_width, &decoded_height, source_path.c_str(), &error_message); + if (read_result != TINYEXR_SUCCESS) + { + ADD_FAILURE() << "TinyEXR failed to decode the test source: " << (error_message ? error_message : "unknown error"); + if (error_message) + FreeEXRErrorMessage(error_message); + std::error_code error; + std::filesystem::remove(source_path, error); + return; + } + + ASSERT_NE(decoded, nullptr); + EXPECT_EQ(decoded_width, width); + EXPECT_EQ(decoded_height, height); + EXPECT_TRUE(EnvironmentMapImporter::IsSupportedEquirectangularSource(decoded_width, decoded_height, decoded)); + for (size_t index = 0; index < source.size(); ++index) + EXPECT_FLOAT_EQ(decoded[index], source[index]) << "component " << index; + + std::free(decoded); + std::error_code error; + std::filesystem::remove(source_path, error); +} + TEST(EnvironmentMapCookingTest, SourceValidationRejectsMalformedAndOversizedInputs) { std::array pixels = {}; diff --git a/ZEngine/tests/Rendering/TextureImporterTest.cpp b/ZEngine/tests/Rendering/TextureImporterTest.cpp index bcc494486..b5701cac9 100644 --- a/ZEngine/tests/Rendering/TextureImporterTest.cpp +++ b/ZEngine/tests/Rendering/TextureImporterTest.cpp @@ -16,9 +16,9 @@ TEST(TextureImporterTest, CanImportClaimsAllEightRasterExtensions) TEST(TextureImporterTest, CanImportDoesNotClaimEnvironmentMapOrContainerFormats) { TextureImporter importer; - // HDR is claimed by EnvironmentMapImporter; EXR and KTX variants remain + // HDR and EXR are claimed by EnvironmentMapImporter; KTX variants remain // recognized by AssetRegistry::InferTypeFromExtension but unsupported until - // their dedicated decoders are introduced. None belong to this importer. + // their dedicated decoder is introduced. None belong to this importer. const char* unclaimed[] = {"hdr", "exr", "ktx", "ktx2", "zenvmap", "glb", "fbx", "obj"}; for (const char* ext : unclaimed) EXPECT_FALSE(importer.CanImport(ext)) << ext; diff --git a/dependencies.cmake b/dependencies.cmake index 486b88945..1f25e5a59 100644 --- a/dependencies.cmake +++ b/dependencies.cmake @@ -15,6 +15,13 @@ FetchContent_Declare( SOURCE_DIR ${FETCHCONTENT_BASE_DIR}/stb ) +FetchContent_Declare( + tinyexr + GIT_REPOSITORY https://github.com/syoyo/tinyexr.git + GIT_TAG v1.0.13 + GIT_SHALLOW TRUE + ) + FetchContent_Declare( glfw3 GIT_REPOSITORY https://github.com/glfw/glfw.git @@ -242,6 +249,17 @@ FetchContent_MakeAvailable( fastgltf ) +# TinyEXR's bundled CMake project declares another target named miniz. Fetch +# only its source instead, then build it against the Miniz target already used +# by the engine. +FetchContent_GetProperties(tinyexr) +if(NOT tinyexr_POPULATED) + FetchContent_Populate(tinyexr) +endif() +add_library(tinyexr STATIC ${tinyexr_SOURCE_DIR}/tinyexr.cc) +target_include_directories(tinyexr PUBLIC ${tinyexr_SOURCE_DIR}) +target_link_libraries(tinyexr PRIVATE miniz ${CMAKE_DL_LIBS}) + foreach(_spirv_target IN ITEMS SPIRV-Tools SPIRV-Tools-static SPIRV-Tools-shared SPIRV-Tools-opt SPIRV-Tools-reduce SPIRV-Tools-link @@ -299,6 +317,7 @@ target_link_libraries(External_libs nlohmann_json::nlohmann_json miniz fastgltf::fastgltf + tinyexr ufbx meshoptimizer freetype From 91826decf3dad5938b017ce1655828551d854d44 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Wed, 16 Sep 2026 13:43:20 +0900 Subject: [PATCH 3/4] fix(import): validate HDR environment artifacts --- ZEngine/ZEngine/Importers/AssetCodec.cpp | 21 ++++++++-- .../Importers/EnvironmentMapImporter.cpp | 38 ++++++++++--------- .../Rendering/Renderers/GraphicRenderer.cpp | 2 +- .../Rendering/EnvironmentMapCooking_test.cpp | 2 + 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/ZEngine/ZEngine/Importers/AssetCodec.cpp b/ZEngine/ZEngine/Importers/AssetCodec.cpp index 9da7ea850..1955c3aee 100644 --- a/ZEngine/ZEngine/Importers/AssetCodec.cpp +++ b/ZEngine/ZEngine/Importers/AssetCodec.cpp @@ -21,6 +21,19 @@ using ZEngine::Core::VFS::VFSPath; namespace ZEngine::Importers::AssetCodec { + namespace + { + bool HasExactEnvironmentMapFileSize(std::ifstream& input, const EnvironmentMapFileHeader& header) + { + input.seekg(0, std::ios::end); + if (!input.good()) + return false; + + const std::streamoff expected_file_size = static_cast(header.HeaderByteSize) + static_cast(header.BufferByteSize); + return input.tellg() == expected_file_size; + } + } // namespace + // Write data atomically via VFS: open .tmp, write, flush, close, rename to out_path. static bool WriteVFS(Core::VFS::IVFSContext* vfs, const VFSPath& out_path, const std::string& data) { @@ -366,13 +379,13 @@ namespace ZEngine::Importers::AssetCodec if (!in.good() || !IsEnvironmentMapFileHeaderValid(header)) return false; - in.seekg(0, std::ios::end); - const std::streamoff expected_file_size = static_cast(header.HeaderByteSize) + static_cast(header.BufferByteSize); - if (in.tellg() != expected_file_size) + if (!HasExactEnvironmentMapFileSize(in, header)) return false; in.seekg(static_cast(header.HeaderByteSize), std::ios::beg); Rendering::Buffers::Bitmap cubemap = Rendering::Buffers::Bitmap::Create(static_cast(header.FaceWidth), static_cast(header.FaceHeight), static_cast(header.LayerCount), static_cast(header.Channel), Rendering::Buffers::BitmapFormat::Float, Rendering::Buffers::BitmapType::CubeMap); + if (!cubemap.Buffer) + return false; in.read(reinterpret_cast(cubemap.Buffer), static_cast(header.BufferByteSize)); if (!in.good()) return false; @@ -387,7 +400,7 @@ namespace ZEngine::Importers::AssetCodec if (!in.is_open()) return false; in.read(reinterpret_cast(&out_header), sizeof(EnvironmentMapFileHeader)); - return in.good() && IsEnvironmentMapFileHeaderValid(out_header); + return in.good() && IsEnvironmentMapFileHeaderValid(out_header) && HasExactEnvironmentMapFileSize(in, out_header); } Core::VFS::VFSResult SerializeEnvironmentMapFileVFS(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& out_path, const Rendering::Buffers::Bitmap& cubemap, const EnvironmentMapCookMetadata& metadata) diff --git a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp index 6e86e8d34..6e6c25808 100644 --- a/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp +++ b/ZEngine/ZEngine/Importers/EnvironmentMapImporter.cpp @@ -21,7 +21,9 @@ namespace ZEngine::Importers { namespace { - void AddImportSetting(Core::VFS::MetaFileData& meta, const char* key, const char* value) + constexpr int k_rgba_channel_count = 4; + + void AddImportSetting(Core::VFS::MetaFileData& meta, const char* key, const char* value) { if (meta.SettingsCount >= Core::VFS::META_MAX_SETTINGS) return; @@ -29,6 +31,14 @@ namespace ZEngine::Importers std::snprintf(setting.Key, sizeof(setting.Key), "%s", key); std::snprintf(setting.Value, sizeof(setting.Value), "%s", value); } + + void FreeDecodedPixels(float* pixels, bool is_exr) + { + if (is_exr) + std::free(pixels); + else + stbi_image_free(pixels); + } } // namespace void EnvironmentMapImporter::Initialize(Core::Memory::ArenaAllocator* arena) @@ -69,8 +79,8 @@ namespace ZEngine::Importers Core::VFS::VFSResult EnvironmentMapImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) { - // stb_image works on the filesystem, while the source identity and cooked - // artifact remain VFS paths. Resolve the source relative to its workspace. + // The decoders work on the filesystem, while the source identity and + // cooked artifact remain VFS paths. Resolve the source relative to its workspace. char native[MAX_FILE_PATH_COUNT] = {}; const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; if (working_space && working_space[0] != '\0') @@ -78,7 +88,7 @@ namespace ZEngine::Importers else path.ToNative(native, sizeof(native)); - int width = 0, height = 0, channel = 0; + int width = 0, height = 0; float* image_data = nullptr; const bool is_exr = path.Extension().Equals(".exr"); if (is_exr) @@ -89,15 +99,13 @@ namespace ZEngine::Importers ZENGINE_CORE_ERROR("EnvironmentMapImporter: failed to load EXR '{}': {}", native, error_message ? error_message : "unknown decoder error") if (error_message) FreeEXRErrorMessage(error_message); - if (image_data) - std::free(image_data); + FreeDecodedPixels(image_data, true); return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); } - channel = STBI_rgb_alpha; } else { - image_data = const_cast(stbi_loadf(native, &width, &height, &channel, STBI_rgb_alpha)); + image_data = stbi_loadf(native, &width, &height, nullptr, k_rgba_channel_count); } if (!image_data) { @@ -107,20 +115,14 @@ namespace ZEngine::Importers if (!IsSupportedEquirectangularSource(width, height, image_data)) { - if (is_exr) - std::free(image_data); - else - stbi_image_free(image_data); - ZENGINE_CORE_ERROR("EnvironmentMapImporter: '{}' must be a finite, non-negative 2:1 HDR equirectangular image with a face size no larger than {}", native, AssetCodec::ENVIRONMENT_MAP_MAX_FACE_SIZE) + FreeDecodedPixels(image_data, is_exr); + ZENGINE_CORE_ERROR("EnvironmentMapImporter: '{}' must be a finite, non-negative 2:1 HDRI equirectangular image with a face size no larger than {}", native, AssetCodec::ENVIRONMENT_MAP_MAX_FACE_SIZE) return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::InvalidPath); } Core::Memory::TLSFSlab* slab = Helpers::GetWorkerSlab(); - Bitmap equirect = Bitmap::FromData(width, height, 1, STBI_rgb_alpha, BitmapFormat::Float, BitmapType::Texture2D, image_data); - if (is_exr) - std::free(image_data); - else - stbi_image_free(image_data); + Bitmap equirect = Bitmap::FromData(width, height, 1, k_rgba_channel_count, BitmapFormat::Float, BitmapType::Texture2D, image_data); + FreeDecodedPixels(image_data, is_exr); Bitmap cubemap = BitmapConvert::EquirectToCubemap(equirect, slab); diff --git a/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp b/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp index 805fdc0c3..53bb23a95 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp @@ -527,7 +527,7 @@ namespace ZEngine::Rendering::Renderers const Scenes::AtmosphereStaticResources atmosphere = m_sky_environment.GetActiveBakeAtmosphere(); const bool owns_atmosphere = m_sky_environment.ActiveBakeOwnsAtmosphere(); const EnvironmentLightingResources lighting = m_sky_environment.GetActiveBakeLighting(); - const Scenes::SkyEnvironmentBakeResult result = m_sky_environment.CompleteBake(revision, source_radiance, false); + m_sky_environment.CompleteBake(revision, source_radiance, false); DiscardSkyResources({.Atmosphere = owns_atmosphere ? atmosphere : Scenes::AtmosphereStaticResources{}, .SourceRadiance = source_radiance, .Lighting = lighting}); ZENGINE_CORE_INFO("[SkyEnvironment] Cancelled stale revision {} between GPU bake stages", revision) StartPendingSkyBake(); diff --git a/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp b/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp index cb45b82a6..b142deea7 100644 --- a/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp +++ b/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp @@ -167,6 +167,8 @@ TEST(EnvironmentMapCookingTest, DeserializerRejectsTruncatedCookedArtifact) Bitmap corrupt = {}; EXPECT_FALSE(DeserializeEnvironmentMapFile(artifact_path.c_str(), corrupt)); + EnvironmentMapFileHeader truncated_header = {}; + EXPECT_FALSE(ReadEnvironmentMapFileHeader(artifact_path.c_str(), truncated_header)); std::error_code error; std::filesystem::remove(artifact_path, error); } From 43ddb47efde51bf3f247838ece4a409a582242d8 Mon Sep 17 00:00:00 2001 From: Jean Philippe Date: Wed, 16 Sep 2026 17:45:34 +0900 Subject: [PATCH 4/4] fix(test): use narrow paths for EXR fixtures --- .../tests/Rendering/EnvironmentMapCooking_test.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp b/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp index b142deea7..365c03f0c 100644 --- a/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp +++ b/ZEngine/tests/Rendering/EnvironmentMapCooking_test.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -57,12 +58,13 @@ TEST(EnvironmentMapCookingTest, TinyExrDecodesAFloatEquirectangularSource) constexpr int width = 4; constexpr int height = 2; const std::filesystem::path source_path = std::filesystem::temp_directory_path() / "zengine_environment_map_cooking_test.exr"; + const std::string source_file = source_path.string(); std::array source = {}; for (size_t index = 0; index < source.size(); ++index) source[index] = static_cast(index) * 0.25f; const char* error_message = nullptr; - const int write_result = SaveEXR(source.data(), width, height, 4, 0, source_path.c_str(), &error_message); + const int write_result = SaveEXR(source.data(), width, height, 4, 0, source_file.c_str(), &error_message); if (write_result != TINYEXR_SUCCESS) { ADD_FAILURE() << "TinyEXR failed to write the test source: " << (error_message ? error_message : "unknown error"); @@ -75,7 +77,7 @@ TEST(EnvironmentMapCookingTest, TinyExrDecodesAFloatEquirectangularSource) int decoded_width = 0; int decoded_height = 0; error_message = nullptr; - const int read_result = LoadEXR(&decoded, &decoded_width, &decoded_height, source_path.c_str(), &error_message); + const int read_result = LoadEXR(&decoded, &decoded_width, &decoded_height, source_file.c_str(), &error_message); if (read_result != TINYEXR_SUCCESS) { ADD_FAILURE() << "TinyEXR failed to decode the test source: " << (error_message ? error_message : "unknown error"); @@ -143,6 +145,7 @@ TEST(EnvironmentMapCookingTest, HeaderValidationRejectsWrongContractsAndStaleSou TEST(EnvironmentMapCookingTest, DeserializerRejectsTruncatedCookedArtifact) { const std::filesystem::path artifact_path = std::filesystem::temp_directory_path() / "zengine_environment_map_cooking_test.zenvmap"; + const std::string artifact_file = artifact_path.string(); const EnvironmentMapFileHeader header = MakeValidHeader(); const std::vector payload(static_cast(header.BufferByteSize) / sizeof(float), 0.25f); @@ -154,7 +157,7 @@ TEST(EnvironmentMapCookingTest, DeserializerRejectsTruncatedCookedArtifact) } Bitmap decoded = {}; - ASSERT_TRUE(DeserializeEnvironmentMapFile(artifact_path.c_str(), decoded)); + ASSERT_TRUE(DeserializeEnvironmentMapFile(artifact_file.c_str(), decoded)); EXPECT_EQ(decoded.Type, BitmapType::CubeMap); EXPECT_EQ(decoded.Width, 2); EXPECT_EQ(decoded.Layers, 6); @@ -166,9 +169,9 @@ TEST(EnvironmentMapCookingTest, DeserializerRejectsTruncatedCookedArtifact) } Bitmap corrupt = {}; - EXPECT_FALSE(DeserializeEnvironmentMapFile(artifact_path.c_str(), corrupt)); + EXPECT_FALSE(DeserializeEnvironmentMapFile(artifact_file.c_str(), corrupt)); EnvironmentMapFileHeader truncated_header = {}; - EXPECT_FALSE(ReadEnvironmentMapFileHeader(artifact_path.c_str(), truncated_header)); + EXPECT_FALSE(ReadEnvironmentMapFileHeader(artifact_file.c_str(), truncated_header)); std::error_code error; std::filesystem::remove(artifact_path, error); }