diff --git a/Package.resolved b/Package.resolved index 10fae4bdb..25c0dcb39 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "dcc639b6cdf875204fc0d722e0eae8f11a5b19fb8517998bd776a9b76b48c3e8", + "originHash" : "8b51a9ec068537ab57ce9b8034b5b84a02a4697e4a6be491954e5fbda7e5783b", "pins" : [ { "identity" : "async-http-client", @@ -216,6 +216,15 @@ "revision" : "395a77f0aa927f0ff73941d7ac35f2b46d47c9db", "version" : "1.6.3" } + }, + { + "identity" : "zstd", + "kind" : "remoteSourceControl", + "location" : "https://github.com/facebook/zstd.git", + "state" : { + "revision" : "f8745da6ff1ad1e7bab384bd1f9d742439278e99", + "version" : "1.5.7" + } } ], "version" : 3 diff --git a/Package.swift b/Package.swift index eeef8042e..d3e1c7dba 100644 --- a/Package.swift +++ b/Package.swift @@ -47,6 +47,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-system.git", from: "1.4.0"), .package(url: "https://github.com/swiftlang/swift-docc-plugin", from: "1.1.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.36.0"), + .package(url: "https://github.com/facebook/zstd.git", exact: "1.5.7"), ], targets: [ .target( @@ -143,17 +144,26 @@ let package = Package( name: "ContainerizationArchiveTests", dependencies: [ "ContainerizationArchive" + ], + resources: [ + .copy("Resources/test.tar.zst") ] ), .target( name: "CArchive", - dependencies: [], + dependencies: [ + .product(name: "libzstd", package: "zstd") + ], path: "Sources/ContainerizationArchive/CArchive", + sources: [ + "archive_swift_bridge.c" + ], cSettings: [ .define( "PLATFORM_CONFIG_H", to: "\"config_darwin.h\"", .when(platforms: [.iOS, .macOS, .macCatalyst, .watchOS, .driverKit, .tvOS])), .define("PLATFORM_CONFIG_H", to: "\"config_linux.h\"", .when(platforms: [.linux])), + .unsafeFlags(["-fno-modules"]), ], linkerSettings: [ .linkedLibrary("z"), diff --git a/Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift b/Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift index 66f4e4196..4e588e8d3 100644 --- a/Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift +++ b/Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift @@ -94,6 +94,8 @@ public struct EXT4Unpacker: Unpacker { compression = .none case MediaTypes.imageLayerGzip, MediaTypes.dockerImageLayerGzip: compression = .gzip + case MediaTypes.imageLayerZstd, MediaTypes.dockerImageLayerZstd: + compression = .zstd default: throw ContainerizationError(.unsupported, message: "media type \(layer.mediaType) not supported.") } diff --git a/Sources/ContainerizationArchive/ArchiveReader.swift b/Sources/ContainerizationArchive/ArchiveReader.swift index 85ba6d159..930481526 100644 --- a/Sources/ContainerizationArchive/ArchiveReader.swift +++ b/Sources/ContainerizationArchive/ArchiveReader.swift @@ -15,9 +15,11 @@ //===----------------------------------------------------------------------===// import CArchive +import ContainerizationError import ContainerizationOS import Foundation import SystemPackage +import libzstd /// A protocol for reading data in chunks, compatible with both `InputStream` and zero-allocation archive readers. public protocol ReadableStream { @@ -53,13 +55,32 @@ public final class ArchiveReader { var underlying: OpaquePointer? /// The file handle associated with the archive file being read. let fileHandle: FileHandle? + /// Temporary decompressed file URL if the input was zstd-compressed + private var tempDecompressedFile: URL? /// Initializes an `ArchiveReader` to read from a specified file URL with an explicit `Format` and `Filter`. /// Note: This method must be used when it is known that the archive at the specified URL follows the specified /// `Format` and `Filter`. public convenience init(format: Format, filter: Filter, file: URL) throws { - let fileHandle = try FileHandle(forReadingFrom: file) - try self.init(format: format, filter: filter, fileHandle: fileHandle) + // If filter is zstd, decompress it and use filter .none + let fileToRead: URL + let tempFile: URL? + let actualFilter: Filter + + if filter == .zstd { + let decompressed = try Self.decompressZstd(file) + tempFile = decompressed + fileToRead = decompressed + actualFilter = .none + } else { + tempFile = nil + fileToRead = file + actualFilter = filter + } + + let fileHandle = try FileHandle(forReadingFrom: fileToRead) + try self.init(format: format, filter: actualFilter, fileHandle: fileHandle) + self.tempDecompressedFile = tempFile } /// Initializes an `ArchiveReader` to read from the provided file descriptor with an explicit `Format` and `Filter`. @@ -82,8 +103,19 @@ public final class ArchiveReader { /// Initialize the `ArchiveReader` to read from a specified file URL /// by trying to auto determine the archives `Format` and `Filter`. public init(file: URL) throws { + self.underlying = archive_read_new() - let fileHandle = try FileHandle(forReadingFrom: file) + + // Try to decompress as zstd first, fall back to original if it fails + let fileToRead: URL + if let decompressed = try? Self.decompressZstd(file) { + self.tempDecompressedFile = decompressed + fileToRead = decompressed + } else { + fileToRead = file + } + + let fileHandle = try FileHandle(forReadingFrom: fileToRead) self.fileHandle = fileHandle try archive_read_support_filter_all(underlying) .checkOk(elseThrow: .failedToDetectFilter) @@ -94,9 +126,84 @@ public final class ArchiveReader { .checkOk(elseThrow: { .unableToOpenArchive($0) }) } + /// Decompress a zstd file to a temporary location + private static func decompressZstd(_ source: URL) throws -> URL { + guard let inputStream = InputStream(url: source) else { + throw ArchiveError.noUnderlyingArchive + } + inputStream.open() + defer { inputStream.close() } + + // Create temp file into which the source zstd archived is decompressed + guard let tempDir = createTemporaryDirectory(baseName: "zstd-decompress") else { + throw ArchiveError.failedToDetectFormat + } + + let tempFile = tempDir.appendingPathComponent( + source.deletingPathExtension().lastPathComponent + ) + + guard let outputStream = OutputStream(url: tempFile, append: false) else { + throw ArchiveError.noUnderlyingArchive + } + outputStream.open() + defer { outputStream.close() } + + // Use streaming decompression since content size may be unknown + guard let dstream = ZSTD_createDStream() else { + throw ArchiveError.failedToDetectFormat + } + defer { ZSTD_freeDStream(dstream) } + + let initResult = ZSTD_initDStream(dstream) + guard ZSTD_isError(initResult) == 0 else { + throw ArchiveError.failedToDetectFormat + } + + let inputBufferSize = ZSTD_DStreamInSize() + let outputBufferSize = ZSTD_DStreamOutSize() + + var inputBuffer = [UInt8](repeating: 0, count: inputBufferSize) + + while case let amount = inputStream.read(&inputBuffer, maxLength: inputBufferSize), amount > 0 { + try inputBuffer.withUnsafeBufferPointer { ptr in + var input = ZSTD_inBuffer( + src: ptr.baseAddress, + size: amount, + pos: 0 + ) + while input.pos < input.size { + var outputBuffer = [UInt8](repeating: 0, count: outputBufferSize) + var decompressedBytes = 0 + try outputBuffer.withUnsafeMutableBytes { outputBytes in + var output = ZSTD_outBuffer( + dst: outputBytes.baseAddress, + size: outputBufferSize, + pos: 0 + ) + let result = ZSTD_decompressStream(dstream, &output, &input) + guard ZSTD_isError(result) == 0 else { + throw ArchiveError.failedToDetectFormat + } + decompressedBytes = output.pos + } + if decompressedBytes > 0 { + outputStream.write(outputBuffer, maxLength: decompressedBytes) + } + } + } + } + return tempFile + } + deinit { archive_read_free(underlying) try? fileHandle?.close() + + // Clean up temp decompressed file + if let tempFile = tempDecompressedFile { + try? FileManager.default.removeItem(at: tempFile.deletingLastPathComponent()) + } } } diff --git a/Sources/ContainerizationArchive/ArchiveWriterConfiguration.swift b/Sources/ContainerizationArchive/ArchiveWriterConfiguration.swift index 0f591edb0..b95f1ff5d 100644 --- a/Sources/ContainerizationArchive/ArchiveWriterConfiguration.swift +++ b/Sources/ContainerizationArchive/ArchiveWriterConfiguration.swift @@ -174,6 +174,7 @@ public enum Filter: String, Sendable { case lzop case grzip case lz4 + case zstd internal var code: CInt { switch self { @@ -190,6 +191,7 @@ public enum Filter: String, Sendable { case .lzop: return ARCHIVE_FILTER_LZOP case .grzip: return ARCHIVE_FILTER_GRZIP case .lz4: return ARCHIVE_FILTER_LZ4 + case .zstd: return ARCHIVE_FILTER_ZSTD } } } diff --git a/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift b/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift index 074e56747..c7c0e563a 100644 --- a/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift +++ b/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift @@ -657,4 +657,58 @@ struct ArchiveReaderTests { _ = try reader.extractContents(to: extractDir) } } + + // MARK: - Zstd Compression Tests + + @Test func readZstdCompressedArchive() throws { + guard let resourceURL = Bundle.module.url(forResource: "test", withExtension: "tar.zst") else { + Issue.record("Test resource test.tar.zst not found") + return + } + + let extractDir = try createExtractionDirectory(name: "zstd-test") + defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) } + + // Test with explicit filter + let reader = try ArchiveReader(format: .paxRestricted, filter: .zstd, file: resourceURL) + let rejectedPaths = try reader.extractContents(to: extractDir) + + #expect(rejectedPaths.isEmpty, "No paths should be rejected") + + // Check extracted files + let testFile = extractDir.appendingPathComponent("test.txt") + let file2 = extractDir.appendingPathComponent("file2.txt") + + #expect(FileManager.default.fileExists(atPath: testFile.path), "test.txt should exist") + #expect(FileManager.default.fileExists(atPath: file2.path), "file2.txt should exist") + + let testContent = try String(contentsOf: testFile, encoding: .utf8) + #expect(testContent == "Hello from zstd compressed archive", "Content should match") + + let file2Content = try String(contentsOf: file2, encoding: .utf8) + #expect(file2Content == "Another file", "Content should match") + } + + @Test func readZstdCompressedArchiveAutoDetect() throws { + guard let resourceURL = Bundle.module.url(forResource: "test", withExtension: "tar.zst") else { + Issue.record("Test resource test.tar.zst not found") + return + } + + let extractDir = try createExtractionDirectory(name: "zstd-auto-test") + defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) } + + // Test with auto-detect + let reader = try ArchiveReader(file: resourceURL) + let rejectedPaths = try reader.extractContents(to: extractDir) + + #expect(rejectedPaths.isEmpty, "No paths should be rejected") + + // Check extracted files + let testFile = extractDir.appendingPathComponent("test.txt") + #expect(FileManager.default.fileExists(atPath: testFile.path), "test.txt should exist") + + let testContent = try String(contentsOf: testFile, encoding: .utf8) + #expect(testContent == "Hello from zstd compressed archive", "Content should match") + } } diff --git a/Tests/ContainerizationArchiveTests/Resources/test.tar.zst b/Tests/ContainerizationArchiveTests/Resources/test.tar.zst new file mode 100644 index 000000000..da79af3fb Binary files /dev/null and b/Tests/ContainerizationArchiveTests/Resources/test.tar.zst differ diff --git a/vminitd/Package.resolved b/vminitd/Package.resolved index dbbf216e7..efd034578 100644 --- a/vminitd/Package.resolved +++ b/vminitd/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "4038f23a0edd12a9ff5bf2cae9327f6ac704134c6d503e0fd08ab254c601cb75", + "originHash" : "db3e9fbb73707e38ad14f86a67eb82b8c1a92edeb98b8c6747530a33f8d87125", "pins" : [ { "identity" : "async-http-client", @@ -198,6 +198,15 @@ "revision" : "395a77f0aa927f0ff73941d7ac35f2b46d47c9db", "version" : "1.6.3" } + }, + { + "identity" : "zstd", + "kind" : "remoteSourceControl", + "location" : "https://github.com/facebook/zstd.git", + "state" : { + "revision" : "f8745da6ff1ad1e7bab384bd1f9d742439278e99", + "version" : "1.5.7" + } } ], "version" : 3