Add stream-based copyIn/copyOut for tar stream cp support

Sources/Containerization/LinuxContainer.swift

@@ -1459,6 +1459,189 @@ extension LinuxContainer {
             }
         }
     }
+
+    /// Copy the contents of a tar stream into the container.
+    ///
+    /// The `archive` file handle is expected to yield a tar stream, either
+    /// uncompressed or compressed (gzip/bzip2/xz), matching `docker cp -` /
+    /// `podman cp -` input semantics. The bytes are streamed directly to the
+    /// guest, which extracts them in place honoring the ownership, mode and
+    /// symlink metadata recorded in the tar headers. No intermediate files are
+    /// created on the host, so host-side permission and path-length constraints
+    /// do not apply.
+    ///
+    /// - Parameters:
+    ///   - archive: A file handle yielding the tar stream to extract.
+    ///   - destination: The guest directory to extract the archive into.
+    ///   - createParents: Create parent directories of `destination` if missing.
+    ///   - chunkSize: The transfer chunk size in bytes.
+    public func copyIn(
+        archive: FileHandle,
+        to destination: URL,
+        createParents: Bool = true,
+        chunkSize: Int = defaultCopyChunkSize
+    ) async throws {
+        try await self.state.withLock {
+            let state = try $0.startedState("copyIn")
+
+            let root = self.root
+            let destinationPath = destination.path
+            let port = self.hostVsockPorts.allocate()
+            // Deferred LIFO: listener back first, then the port number.
+            defer { self.hostVsockPorts.release(port) }
+            let listener = try state.vm.listen(port)
+            defer { try? listener.finish() }
+
+            try await withThrowingTaskGroup(of: Void.self) { group in
+                group.addTask {
+                    try await state.vm.withAgent { agent in
+                        guard let vminitd = agent as? Vminitd else {
+                            throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent")
+                        }
+                        try await vminitd.copy(
+                            direction: .copyIn,
+                            root: root,
+                            path: destinationPath,
+                            vsockPort: port,
+                            createParents: createParents,
+                            isArchive: true
+                        )
+                    }
+                }
+
+                group.addTask {
+                    guard let conn = await listener.first(where: { _ in true }) else {
+                        throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established")
+                    }
+                    try listener.finish()
+
+                    try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, any Error>) in
+                        self.copyQueue.async {
+                            do {
+                                defer { conn.closeFile() }
+                                try Self.spliceFd(
+                                    from: archive.fileDescriptor,
+                                    to: conn.fileDescriptor,
+                                    chunkSize: chunkSize,
+                                    label: "copyIn"
+                                )
+                                continuation.resume()
+                            } catch {
+                                continuation.resume(throwing: error)
+                            }
+                        }
+                    }
+                }
+
+                try await group.waitForAll()
+            }
+        }
+    }
+
+    /// Stream the contents of a container path as a tar archive to a file handle.
+    ///
+    /// Matches `docker cp CONTAINER:/path -` / `podman cp` output semantics: the
+    /// output is always a tar archive, even for a single regular file, and
+    /// preserves the ownership and mode recorded in the guest filesystem. The
+    /// guest performs the archiving natively and streams the result directly;
+    /// no intermediate files are created on the host.
+    ///
+    /// - Parameters:
+    ///   - source: The guest path (file or directory) to archive.
+    ///   - archive: A file handle the tar stream is written to.
+    ///   - chunkSize: The transfer chunk size in bytes.
+    public func copyOut(
+        from source: URL,
+        to archive: FileHandle,
+        chunkSize: Int = defaultCopyChunkSize
+    ) async throws {
+        try await self.state.withLock {
+            let state = try $0.startedState("copyOut")
+
+            let root = self.root
+            let sourcePath = source.path
+            let port = self.hostVsockPorts.allocate()
+            // Deferred LIFO: listener back first, then the port number.
+            defer { self.hostVsockPorts.release(port) }
+            let listener = try state.vm.listen(port)
+            defer { try? listener.finish() }
+
+            let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self)
+
+            try await withThrowingTaskGroup(of: Void.self) { group in
+                group.addTask {
+                    try await state.vm.withAgent { agent in
+                        guard let vminitd = agent as? Vminitd else {
+                            throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent")
+                        }
+                        try await vminitd.copy(
+                            direction: .copyOut,
+                            root: root,
+                            path: sourcePath,
+                            vsockPort: port,
+                            isArchive: true,
+                            onMetadata: { meta in
+                                metadataCont.yield(meta)
+                                metadataCont.finish()
+                            }
+                        )
+                    }
+                }
+
+                group.addTask {
+                    // Wait for the guest to report metadata (and thus that the
+                    // source exists) before accepting the data connection.
+                    _ = await metadataStream.first(where: { _ in true })
+
+                    guard let conn = await listener.first(where: { _ in true }) else {
+                        throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established")
+                    }
+                    try listener.finish()
+
+                    try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, any Error>) in
+                        self.copyQueue.async {
+                            do {
+                                defer { conn.closeFile() }
+                                try Self.spliceFd(
+                                    from: conn.fileDescriptor,
+                                    to: archive.fileDescriptor,
+                                    chunkSize: chunkSize,
+                                    label: "copyOut"
+                                )
+                                continuation.resume()
+                            } catch {
+                                continuation.resume(throwing: error)
+                            }
+                        }
+                    }
+                }
+
+                try await group.waitForAll()
+            }
+        }
+    }
+
+    /// Copy raw bytes from one file descriptor to another until EOF.
+    private static func spliceFd(from srcFd: Int32, to dstFd: Int32, chunkSize: Int, label: String) throws {
+        var buf = [UInt8](repeating: 0, count: chunkSize)
+        while true {
+            let n = read(srcFd, &buf, buf.count)
+            if n == 0 { break }
+            guard n > 0 else {
+                throw ContainerizationError(.internalError, message: "\(label): read error: \(String(cString: strerror(errno)))")
+            }
+            var written = 0
+            while written < n {
+                let w = buf.withUnsafeBytes { ptr in
+                    write(dstFd, ptr.baseAddress! + written, n - written)
+                }
+                guard w > 0 else {
+                    throw ContainerizationError(.internalError, message: "\(label): vsock write error: \(String(cString: strerror(errno)))")
+                }
+                written += w
+            }
+        }
+    }
 }
 
 extension VirtualMachineInstance {

Sources/ContainerizationArchive/ArchiveReader.swift

@@ -106,6 +106,29 @@ public final class ArchiveReader {
             .checkOk(elseThrow: { .unableToOpenArchive($0) })
     }
 
+    /// Initializes an `ArchiveReader` to read from the provided file descriptor,
+    /// auto-detecting the archive `Format` and compression `Filter`.
+    ///
+    /// Use this when the incoming stream's format and compression are not known
+    /// ahead of time, e.g. a tar stream supplied on stdin that may be
+    /// uncompressed or compressed (gzip, bzip2, xz, ...), matching
+    /// `docker cp -` / `podman cp -` input semantics. zstd is not auto-detected
+    /// from a stream because it requires seeking; use
+    /// ``init(format:filter:fileHandle:)`` with ``Filter/zstd`` for that case.
+    public init(fileHandle: FileHandle) throws {
+        self.underlying = archive_read_new()
+        self.fileHandle = fileHandle
+
+        try archive_read_support_filter_all(underlying)
+            .checkOk(elseThrow: .failedToDetectFilter)
+        try archive_read_support_format_all(underlying)
+            .checkOk(elseThrow: .failedToDetectFormat)
+
+        let fd = fileHandle.fileDescriptor
+        try archive_read_open_fd(underlying, fd, 4096)
+            .checkOk(elseThrow: { .unableToOpenArchive($0) })
+    }
+
     /// Initialize the `ArchiveReader` to read from a specified file URL
     /// by trying to auto determine the archives `Format` and `Filter`.
     public init(file: URL) throws {

Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift

@@ -658,6 +658,66 @@ struct ArchiveReaderTests {
         }
     }
 
+    // MARK: - FileHandle Stream Auto-Detect Tests
+
+    /// The stream-based auto-detecting reader must accept an uncompressed tar
+    /// stream, matching `docker cp -` / `podman cp -` input semantics.
+    @Test func readUncompressedTarFromFileHandle() throws {
+        let archiveURL = try createTestArchive(
+            name: "stream-plain",
+            entries: [
+                ("dir/", .directory, nil),
+                ("dir/file.txt", .regular("streamed plain"), nil),
+            ])
+        defer { try? FileManager.default.removeItem(at: archiveURL.deletingLastPathComponent()) }
+
+        let extractDir = try createExtractionDirectory(name: "stream-plain")
+        defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) }
+
+        let fh = try FileHandle(forReadingFrom: archiveURL)
+        defer { try? fh.close() }
+
+        let reader = try ArchiveReader(fileHandle: fh)
+        let rejected = try reader.extractContents(to: extractDir)
+
+        #expect(rejected.isEmpty)
+        let content = try String(contentsOf: extractDir.appendingPathComponent("dir/file.txt"), encoding: .utf8)
+        #expect(content == "streamed plain")
+    }
+
+    /// The stream-based auto-detecting reader must also accept a gzip-compressed
+    /// tar stream (the format the guest emits internally for directory copies),
+    /// without being told the filter up front.
+    @Test func readGzipTarFromFileHandle() throws {
+        let testDirectory = createTemporaryDirectory(baseName: "ArchiveReaderTests")!
+        defer { try? FileManager.default.removeItem(at: testDirectory) }
+        let archiveURL = testDirectory.appendingPathComponent("stream-gzip.tar.gz")
+
+        let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip))
+        try writer.open(file: archiveURL)
+        let entry = WriteEntry()
+        entry.path = "hello.txt"
+        entry.fileType = .regular
+        entry.permissions = 0o644
+        let data = "streamed gzip".data(using: .utf8)!
+        entry.size = numericCast(data.count)
+        try writer.writeEntry(entry: entry, data: data)
+        try writer.finishEncoding()
+
+        let extractDir = try createExtractionDirectory(name: "stream-gzip")
+        defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) }
+
+        let fh = try FileHandle(forReadingFrom: archiveURL)
+        defer { try? fh.close() }
+
+        let reader = try ArchiveReader(fileHandle: fh)
+        let rejected = try reader.extractContents(to: extractDir)
+
+        #expect(rejected.isEmpty)
+        let content = try String(contentsOf: extractDir.appendingPathComponent("hello.txt"), encoding: .utf8)
+        #expect(content == "streamed gzip")
+    }
+
     // MARK: - Zstd Compression Tests
 
     @Test func readZstdCompressedArchive() throws {

vminitd/Sources/VminitdCore/Server+GRPC.swift

@@ -561,7 +561,11 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
             // /proc/self/fd pins the resolved directory against path swaps during extraction.
             return try RootfsResolver.withDirectoryInRoot(dirFd: rootFd, path: path) { destFd in
                 let fileHandle = FileHandle(fileDescriptor: sockFd, closeOnDealloc: false)
-                let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fileHandle)
+                // Auto-detect format and compression filter so both the internal
+                // pax+gzip archives (directory copyIn) and externally supplied tar
+                // streams (uncompressed or compressed, e.g. `docker cp -`) are
+                // accepted.
+                let reader = try ArchiveReader(fileHandle: fileHandle)
                 return try reader.extractContents(to: URL(fileURLWithPath: "/proc/self/fd/\(destFd)"))
             }
         }
@@ -606,10 +610,13 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
             throw RPCError(code: .internalError, message: "copy: failed to stat '\(path)': \(swiftErrno("fstat"))")
         }
         let fileType = s.st_mode & UInt32(S_IFMT)
-        let isArchive = fileType == UInt32(S_IFDIR)
-        guard isArchive || fileType == UInt32(S_IFREG) else {
+        guard fileType == UInt32(S_IFDIR) || fileType == UInt32(S_IFREG) else {
             throw RPCError(code: .invalidArgument, message: "copy: cannot copy out '\(path)': unsupported file type")
         }
+        // `request.isArchive` forces archive output even for a single regular
+        // file, matching `docker cp CONTAINER:/path -` which always emits a tar
+        // stream. Directories are always archived.
+        let isArchive = fileType == UInt32(S_IFDIR) || request.isArchive
         let totalSize: UInt64 = isArchive ? 0 : UInt64(s.st_size)
 
         // Send metadata response BEFORE connecting to vsock, so host knows what to expect.
@@ -632,9 +639,15 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
                 // TODO: archiveDirectory is path-based; running containers can still
                 // race directory copyOut. Close this with an fd-relative archive writer.
                 let source = try FileDescriptorOps.getCanonicalPath(FileDescriptor(rawValue: fd)).string
-                let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip))
+                let filter: Filter = request.isArchive ? Filter.none : Filter.gzip
+                let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: filter))
                 try writer.open(fileDescriptor: sock.fileDescriptor)
-                try writer.archiveDirectory(URL(fileURLWithPath: source))
+                if request.isArchive {
+                    let filePath = FilePath(source)
+                    try writer.archive([filePath], base: filePath.removingLastComponent())
+                } else {
+                    try writer.archiveDirectory(URL(fileURLWithPath: source))
+                }
                 try writer.finishEncoding()
             } else {
                 // Reopen through the pinned fd after confirming this is a regular file.