From 57acd28cb0f53bbe28009f67ea0f9b41f221575f Mon Sep 17 00:00:00 2001 From: John Logan Date: Mon, 2 Feb 2026 10:53:49 -0800 Subject: [PATCH 1/3] Adds zstd decompression for layer content blobs. - Closes apple/container#988. - macOS libarchive is not built with zstd support, so the workaround in ArchiveReader is to attempt to decompress every archive as zstd. If decompression fails, we pass the original archive to libarchive. If it succeeds, we pass the uncompressed archive. - Adds blob media type recognition for zstd to EXT4Unpacker. Tested zstd blob unpack using `image pull tonistiigi/hello-world:zstd-docker`. --- Package.resolved | 11 +- Package.swift | 12 +- .../Image/Unpacker/EXT4Unpacker.swift | 2 + .../ArchiveReader.swift | 105 +++++++++++++++++- .../ArchiveWriterConfiguration.swift | 2 + .../ArchiveReaderTests.swift | 54 +++++++++ .../Resources/test.tar.zst | Bin 0 -> 445 bytes vminitd/Package.resolved | 11 +- 8 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 Tests/ContainerizationArchiveTests/Resources/test.tar.zst diff --git a/Package.resolved b/Package.resolved index 10fae4bdb..25c0dcb39 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "dcc639b6cdf875204fc0d722e0eae8f11a5b19fb8517998bd776a9b76b48c3e8", + "originHash" : "8b51a9ec068537ab57ce9b8034b5b84a02a4697e4a6be491954e5fbda7e5783b", "pins" : [ { "identity" : "async-http-client", @@ -216,6 +216,15 @@ "revision" : "395a77f0aa927f0ff73941d7ac35f2b46d47c9db", "version" : "1.6.3" } + }, + { + "identity" : "zstd", + "kind" : "remoteSourceControl", + "location" : "https://github.com/facebook/zstd.git", + "state" : { + "revision" : "f8745da6ff1ad1e7bab384bd1f9d742439278e99", + "version" : "1.5.7" + } } ], "version" : 3 diff --git a/Package.swift b/Package.swift index eeef8042e..d3e1c7dba 100644 --- a/Package.swift +++ b/Package.swift @@ -47,6 +47,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-system.git", from: "1.4.0"), .package(url: "https://github.com/swiftlang/swift-docc-plugin", from: "1.1.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.36.0"), + .package(url: "https://github.com/facebook/zstd.git", exact: "1.5.7"), ], targets: [ .target( @@ -143,17 +144,26 @@ let package = Package( name: "ContainerizationArchiveTests", dependencies: [ "ContainerizationArchive" + ], + resources: [ + .copy("Resources/test.tar.zst") ] ), .target( name: "CArchive", - dependencies: [], + dependencies: [ + .product(name: "libzstd", package: "zstd") + ], path: "Sources/ContainerizationArchive/CArchive", + sources: [ + "archive_swift_bridge.c" + ], cSettings: [ .define( "PLATFORM_CONFIG_H", to: "\"config_darwin.h\"", .when(platforms: [.iOS, .macOS, .macCatalyst, .watchOS, .driverKit, .tvOS])), .define("PLATFORM_CONFIG_H", to: "\"config_linux.h\"", .when(platforms: [.linux])), + .unsafeFlags(["-fno-modules"]), ], linkerSettings: [ .linkedLibrary("z"), diff --git a/Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift b/Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift index 66f4e4196..4e588e8d3 100644 --- a/Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift +++ b/Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift @@ -94,6 +94,8 @@ public struct EXT4Unpacker: Unpacker { compression = .none case MediaTypes.imageLayerGzip, MediaTypes.dockerImageLayerGzip: compression = .gzip + case MediaTypes.imageLayerZstd, MediaTypes.dockerImageLayerZstd: + compression = .zstd default: throw ContainerizationError(.unsupported, message: "media type \(layer.mediaType) not supported.") } diff --git a/Sources/ContainerizationArchive/ArchiveReader.swift b/Sources/ContainerizationArchive/ArchiveReader.swift index 85ba6d159..717dbd809 100644 --- a/Sources/ContainerizationArchive/ArchiveReader.swift +++ b/Sources/ContainerizationArchive/ArchiveReader.swift @@ -15,9 +15,11 @@ //===----------------------------------------------------------------------===// import CArchive +import ContainerizationError import ContainerizationOS import Foundation import SystemPackage +import libzstd /// A protocol for reading data in chunks, compatible with both `InputStream` and zero-allocation archive readers. public protocol ReadableStream { @@ -53,13 +55,32 @@ public final class ArchiveReader { var underlying: OpaquePointer? /// The file handle associated with the archive file being read. let fileHandle: FileHandle? + /// Temporary decompressed file URL if the input was zstd-compressed + private var tempDecompressedFile: URL? /// Initializes an `ArchiveReader` to read from a specified file URL with an explicit `Format` and `Filter`. /// Note: This method must be used when it is known that the archive at the specified URL follows the specified /// `Format` and `Filter`. public convenience init(format: Format, filter: Filter, file: URL) throws { - let fileHandle = try FileHandle(forReadingFrom: file) - try self.init(format: format, filter: filter, fileHandle: fileHandle) + // If filter is zstd, decompress it and use filter .none + let fileToRead: URL + let tempFile: URL? + let actualFilter: Filter + + if filter == .zstd { + let decompressed = try Self.decompressZstd(file) + tempFile = decompressed + fileToRead = decompressed + actualFilter = .none + } else { + tempFile = nil + fileToRead = file + actualFilter = filter + } + + let fileHandle = try FileHandle(forReadingFrom: fileToRead) + try self.init(format: format, filter: actualFilter, fileHandle: fileHandle) + self.tempDecompressedFile = tempFile } /// Initializes an `ArchiveReader` to read from the provided file descriptor with an explicit `Format` and `Filter`. @@ -82,8 +103,22 @@ public final class ArchiveReader { /// Initialize the `ArchiveReader` to read from a specified file URL /// by trying to auto determine the archives `Format` and `Filter`. public init(file: URL) throws { + print("ArchiveReader.init called with file: \(file.path)") + self.underlying = archive_read_new() - let fileHandle = try FileHandle(forReadingFrom: file) + + // Try to decompress as zstd first, fall back to original if it fails + let fileToRead: URL + if let decompressed = try? Self.decompressZstd(file) { + print("Successfully decompressed zstd to: \(decompressed.path)") + self.tempDecompressedFile = decompressed + fileToRead = decompressed + } else { + print("Not a zstd file or decompression failed, using original") + fileToRead = file + } + + let fileHandle = try FileHandle(forReadingFrom: fileToRead) self.fileHandle = fileHandle try archive_read_support_filter_all(underlying) .checkOk(elseThrow: .failedToDetectFilter) @@ -94,9 +129,73 @@ public final class ArchiveReader { .checkOk(elseThrow: { .unableToOpenArchive($0) }) } + /// Decompress a zstd file to a temporary location + private static func decompressZstd(_ source: URL) throws -> URL { + let inputData = try Data(contentsOf: source) + + // Use streaming decompression since content size may be unknown + guard let dstream = ZSTD_createDStream() else { + throw ArchiveError.failedToDetectFormat + } + defer { ZSTD_freeDStream(dstream) } + + let initResult = ZSTD_initDStream(dstream) + guard ZSTD_isError(initResult) == 0 else { + throw ArchiveError.failedToDetectFormat + } + + var decompressed = Data() + let outputBufferSize = Int(ZSTD_DStreamOutSize()) + + try inputData.withUnsafeBytes { inputBytes in + var input = ZSTD_inBuffer( + src: inputBytes.baseAddress, + size: inputData.count, + pos: 0 + ) + + var outputBuffer = [UInt8](repeating: 0, count: outputBufferSize) + + while input.pos < input.size { + try outputBuffer.withUnsafeMutableBytes { outputBytes in + var output = ZSTD_outBuffer( + dst: outputBytes.baseAddress, + size: outputBufferSize, + pos: 0 + ) + + let result = ZSTD_decompressStream(dstream, &output, &input) + guard ZSTD_isError(result) == 0 else { + throw ArchiveError.failedToDetectFormat + } + + if output.pos > 0 { + decompressed.append(contentsOf: outputBytes.bindMemory(to: UInt8.self).prefix(Int(output.pos))) + } + } + } + } + + // Create temp file + guard let tempDir = createTemporaryDirectory(baseName: "zstd-decompress") else { + throw ArchiveError.failedToDetectFormat + } + + let tempFile = tempDir.appendingPathComponent( + source.deletingPathExtension().lastPathComponent + ) + try decompressed.write(to: tempFile) + return tempFile + } + deinit { archive_read_free(underlying) try? fileHandle?.close() + + // Clean up temp decompressed file + if let tempFile = tempDecompressedFile { + try? FileManager.default.removeItem(at: tempFile.deletingLastPathComponent()) + } } } diff --git a/Sources/ContainerizationArchive/ArchiveWriterConfiguration.swift b/Sources/ContainerizationArchive/ArchiveWriterConfiguration.swift index 0f591edb0..b95f1ff5d 100644 --- a/Sources/ContainerizationArchive/ArchiveWriterConfiguration.swift +++ b/Sources/ContainerizationArchive/ArchiveWriterConfiguration.swift @@ -174,6 +174,7 @@ public enum Filter: String, Sendable { case lzop case grzip case lz4 + case zstd internal var code: CInt { switch self { @@ -190,6 +191,7 @@ public enum Filter: String, Sendable { case .lzop: return ARCHIVE_FILTER_LZOP case .grzip: return ARCHIVE_FILTER_GRZIP case .lz4: return ARCHIVE_FILTER_LZ4 + case .zstd: return ARCHIVE_FILTER_ZSTD } } } diff --git a/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift b/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift index 074e56747..c7c0e563a 100644 --- a/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift +++ b/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift @@ -657,4 +657,58 @@ struct ArchiveReaderTests { _ = try reader.extractContents(to: extractDir) } } + + // MARK: - Zstd Compression Tests + + @Test func readZstdCompressedArchive() throws { + guard let resourceURL = Bundle.module.url(forResource: "test", withExtension: "tar.zst") else { + Issue.record("Test resource test.tar.zst not found") + return + } + + let extractDir = try createExtractionDirectory(name: "zstd-test") + defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) } + + // Test with explicit filter + let reader = try ArchiveReader(format: .paxRestricted, filter: .zstd, file: resourceURL) + let rejectedPaths = try reader.extractContents(to: extractDir) + + #expect(rejectedPaths.isEmpty, "No paths should be rejected") + + // Check extracted files + let testFile = extractDir.appendingPathComponent("test.txt") + let file2 = extractDir.appendingPathComponent("file2.txt") + + #expect(FileManager.default.fileExists(atPath: testFile.path), "test.txt should exist") + #expect(FileManager.default.fileExists(atPath: file2.path), "file2.txt should exist") + + let testContent = try String(contentsOf: testFile, encoding: .utf8) + #expect(testContent == "Hello from zstd compressed archive", "Content should match") + + let file2Content = try String(contentsOf: file2, encoding: .utf8) + #expect(file2Content == "Another file", "Content should match") + } + + @Test func readZstdCompressedArchiveAutoDetect() throws { + guard let resourceURL = Bundle.module.url(forResource: "test", withExtension: "tar.zst") else { + Issue.record("Test resource test.tar.zst not found") + return + } + + let extractDir = try createExtractionDirectory(name: "zstd-auto-test") + defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) } + + // Test with auto-detect + let reader = try ArchiveReader(file: resourceURL) + let rejectedPaths = try reader.extractContents(to: extractDir) + + #expect(rejectedPaths.isEmpty, "No paths should be rejected") + + // Check extracted files + let testFile = extractDir.appendingPathComponent("test.txt") + #expect(FileManager.default.fileExists(atPath: testFile.path), "test.txt should exist") + + let testContent = try String(contentsOf: testFile, encoding: .utf8) + #expect(testContent == "Hello from zstd compressed archive", "Content should match") + } } diff --git a/Tests/ContainerizationArchiveTests/Resources/test.tar.zst b/Tests/ContainerizationArchiveTests/Resources/test.tar.zst new file mode 100644 index 0000000000000000000000000000000000000000..da79af3fbf7a0d75283515d77e850038014db7ac GIT binary patch literal 445 zcmV;u0Yd&LwJ-f-02_S`07er=IB>K*4IOa*OWnRV2mEM0nBr zy8Q>vK!oTBi`GQ0+)|P?U8n&gz+>aqQZfKJ05|{=%nmkSDCq0I5|C9|hCADOxd7fA zA$*`WmC)gw4}@SpGP=7vG-*pY-JNfD-{t;lUJf1XZ|X~3t;jSaNFbnqpa44>{8U!6 z?`U>=e{EZRm_{u$5SZCO<#LG*Fau!b$jk@aooNe2Q#Bot4-AYHAu82HPFT@o5lNoC zw{|ZAGf9(5^e|DHLXU?+pb0sgk2)mM8JQGH9^=JkV*(aDLoRW zeBxggNAtd8Rg|s0d5(L#VH(w&%MEBuvz*qh>m@p4l>WG&Owz;>IUvx25gHk>1N3Bm z7$zVn%$!gg3l^~W*y@NNGUM_;0dQ*3TgBjvTi^uRa)kpFItCydC;@Y_=#~On4)~D` zKy{#-0b*bnF24z8AS>$5TO7pB43UlAV5>VF4$v{&0y`m7q=Iypzu`NrRU0g$%>5m> nycV2cHxY$C Date: Mon, 2 Feb 2026 11:32:51 -0800 Subject: [PATCH 2/3] Remove debug prints. --- Sources/ContainerizationArchive/ArchiveReader.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/Sources/ContainerizationArchive/ArchiveReader.swift b/Sources/ContainerizationArchive/ArchiveReader.swift index 717dbd809..881629801 100644 --- a/Sources/ContainerizationArchive/ArchiveReader.swift +++ b/Sources/ContainerizationArchive/ArchiveReader.swift @@ -103,18 +103,15 @@ public final class ArchiveReader { /// Initialize the `ArchiveReader` to read from a specified file URL /// by trying to auto determine the archives `Format` and `Filter`. public init(file: URL) throws { - print("ArchiveReader.init called with file: \(file.path)") self.underlying = archive_read_new() // Try to decompress as zstd first, fall back to original if it fails let fileToRead: URL if let decompressed = try? Self.decompressZstd(file) { - print("Successfully decompressed zstd to: \(decompressed.path)") self.tempDecompressedFile = decompressed fileToRead = decompressed } else { - print("Not a zstd file or decompression failed, using original") fileToRead = file } From 29a3914af466bb93427f58d1956538ce0052f9ac Mon Sep 17 00:00:00 2001 From: John Logan Date: Mon, 2 Feb 2026 12:52:05 -0800 Subject: [PATCH 3/3] Stream file instead of slurp for decompress. --- .../ArchiveReader.swift | 85 +++++++++++-------- 1 file changed, 48 insertions(+), 37 deletions(-) diff --git a/Sources/ContainerizationArchive/ArchiveReader.swift b/Sources/ContainerizationArchive/ArchiveReader.swift index 881629801..930481526 100644 --- a/Sources/ContainerizationArchive/ArchiveReader.swift +++ b/Sources/ContainerizationArchive/ArchiveReader.swift @@ -128,7 +128,26 @@ public final class ArchiveReader { /// Decompress a zstd file to a temporary location private static func decompressZstd(_ source: URL) throws -> URL { - let inputData = try Data(contentsOf: source) + guard let inputStream = InputStream(url: source) else { + throw ArchiveError.noUnderlyingArchive + } + inputStream.open() + defer { inputStream.close() } + + // Create temp file into which the source zstd archived is decompressed + guard let tempDir = createTemporaryDirectory(baseName: "zstd-decompress") else { + throw ArchiveError.failedToDetectFormat + } + + let tempFile = tempDir.appendingPathComponent( + source.deletingPathExtension().lastPathComponent + ) + + guard let outputStream = OutputStream(url: tempFile, append: false) else { + throw ArchiveError.noUnderlyingArchive + } + outputStream.open() + defer { outputStream.close() } // Use streaming decompression since content size may be unknown guard let dstream = ZSTD_createDStream() else { @@ -141,47 +160,39 @@ public final class ArchiveReader { throw ArchiveError.failedToDetectFormat } - var decompressed = Data() - let outputBufferSize = Int(ZSTD_DStreamOutSize()) - - try inputData.withUnsafeBytes { inputBytes in - var input = ZSTD_inBuffer( - src: inputBytes.baseAddress, - size: inputData.count, - pos: 0 - ) - - var outputBuffer = [UInt8](repeating: 0, count: outputBufferSize) - - while input.pos < input.size { - try outputBuffer.withUnsafeMutableBytes { outputBytes in - var output = ZSTD_outBuffer( - dst: outputBytes.baseAddress, - size: outputBufferSize, - pos: 0 - ) - - let result = ZSTD_decompressStream(dstream, &output, &input) - guard ZSTD_isError(result) == 0 else { - throw ArchiveError.failedToDetectFormat + let inputBufferSize = ZSTD_DStreamInSize() + let outputBufferSize = ZSTD_DStreamOutSize() + + var inputBuffer = [UInt8](repeating: 0, count: inputBufferSize) + + while case let amount = inputStream.read(&inputBuffer, maxLength: inputBufferSize), amount > 0 { + try inputBuffer.withUnsafeBufferPointer { ptr in + var input = ZSTD_inBuffer( + src: ptr.baseAddress, + size: amount, + pos: 0 + ) + while input.pos < input.size { + var outputBuffer = [UInt8](repeating: 0, count: outputBufferSize) + var decompressedBytes = 0 + try outputBuffer.withUnsafeMutableBytes { outputBytes in + var output = ZSTD_outBuffer( + dst: outputBytes.baseAddress, + size: outputBufferSize, + pos: 0 + ) + let result = ZSTD_decompressStream(dstream, &output, &input) + guard ZSTD_isError(result) == 0 else { + throw ArchiveError.failedToDetectFormat + } + decompressedBytes = output.pos } - - if output.pos > 0 { - decompressed.append(contentsOf: outputBytes.bindMemory(to: UInt8.self).prefix(Int(output.pos))) + if decompressedBytes > 0 { + outputStream.write(outputBuffer, maxLength: decompressedBytes) } } } } - - // Create temp file - guard let tempDir = createTemporaryDirectory(baseName: "zstd-decompress") else { - throw ArchiveError.failedToDetectFormat - } - - let tempFile = tempDir.appendingPathComponent( - source.deletingPathExtension().lastPathComponent - ) - try decompressed.write(to: tempFile) return tempFile }