SquashFS: extended file inodes (files over 4 GB, sparse, hard links), sparse blocks, compressor options

Library/DiscUtils.SquashFs/CompressionOptions.cs

@@ -116,13 +116,21 @@ internal static CompressionOptions ReadFrom(Stream stream, SuperBlock block)
 
             case SquashFileSystemCompressionKind.ZLib:
                 {
-                    ThrowIfHasOptions();
-                    return new ZLibCompressionOptions();
+                    ZLibCompressionOptions opts = new();
+                    if (hasOptions)
+                    {
+                        ReadOptions(stream, opts);
+                    }
+                    return opts;
                 }
             case SquashFileSystemCompressionKind.Lzo:
                 {
-                    ThrowIfHasOptions();
-                    return new LzoCompressionOptions();
+                    LzoCompressionOptions opts = new();
+                    if (hasOptions)
+                    {
+                        ReadOptions(stream, opts);
+                    }
+                    return opts;
                 }
             case SquashFileSystemCompressionKind.Lzma:
                 {
@@ -131,8 +139,12 @@ internal static CompressionOptions ReadFrom(Stream stream, SuperBlock block)
                 }
             case SquashFileSystemCompressionKind.ZStd:
                 {
-                    ThrowIfHasOptions();
-                    return new ZStdCompressionOptions();
+                    ZStdCompressionOptions opts = new();
+                    if (hasOptions)
+                    {
+                        ReadOptions(stream, opts);
+                    }
+                    return opts;
                 }
             default:
                 {
@@ -147,6 +159,16 @@ void ThrowIfHasOptions()
                 throw new NotSupportedException($"Unsupported compression options for {block.Compression}");
             }
         }
+
+        static void ReadOptions(Stream stream, CompressionOptions opts)
+        {
+            var size = ReadCompressionSize(stream);
+            if (size < opts.Size)
+            {
+                throw new InvalidDataException($"Invalid {opts.Kind} options size {size}. Expecting at least {opts.Size}");
+            }
+            opts.ReadFrom(stream, size);
+        }
     }
 
     internal static void WriteTo(Stream stream, SuperBlock block, CompressionOptions options)
@@ -307,12 +329,84 @@ public override void WriteTo(Span<byte> buffer)
 /// <summary>
 /// Compression options for ZLib. (No options are supported).
 /// </summary>
-public class ZLibCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.ZLib);
+/// <summary>
+/// Compression options for gzip, present when the image was made with a compression level, window size or
+/// strategy other than the defaults. Decompression does not depend on them.
+/// </summary>
+public class ZLibCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.ZLib)
+{
+    /// <summary>
+    /// The zlib compression level (1-9).
+    /// </summary>
+    public int CompressionLevel { get; private set; } = 9;
+
+    /// <summary>
+    /// The zlib window size (log2, 9-15).
+    /// </summary>
+    public int WindowSize { get; private set; } = 15;
+
+    /// <summary>
+    /// The strategies used, as a bit mask.
+    /// </summary>
+    public int Strategies { get; private set; }
+
+    /// <inheritdoc />
+    public override int Size => 8;
+
+    /// <inheritdoc />
+    public override int ReadFrom(ReadOnlySpan<byte> buffer)
+    {
+        CompressionLevel = EndianUtilities.ToInt32LittleEndian(buffer);
+        WindowSize = EndianUtilities.ToUInt16LittleEndian(buffer[4..]);
+        Strategies = EndianUtilities.ToUInt16LittleEndian(buffer[6..]);
+        return Size;
+    }
+
+    /// <inheritdoc />
+    public override void WriteTo(Span<byte> buffer)
+    {
+        EndianUtilities.WriteBytesLittleEndian(CompressionLevel, buffer);
+        EndianUtilities.WriteBytesLittleEndian((ushort)WindowSize, buffer[4..]);
+        EndianUtilities.WriteBytesLittleEndian((ushort)Strategies, buffer[6..]);
+    }
+}
 
 /// <summary>
 /// Compression options for LZO. (No options are supported).
 /// </summary>
-public class LzoCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.Lzo);
+/// <summary>
+/// Compression options for LZO: the algorithm and, for lzo1x_999, the compression level.
+/// </summary>
+public class LzoCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.Lzo)
+{
+    /// <summary>
+    /// The LZO algorithm (0 lzo1x_1, 1 lzo1x_1_11, 2 lzo1x_1_12, 3 lzo1x_1_15, 4 lzo1x_999).
+    /// </summary>
+    public int Algorithm { get; private set; } = 4;
+
+    /// <summary>
+    /// The compression level for lzo1x_999 (0-9).
+    /// </summary>
+    public int CompressionLevel { get; private set; } = 8;
+
+    /// <inheritdoc />
+    public override int Size => 8;
+
+    /// <inheritdoc />
+    public override int ReadFrom(ReadOnlySpan<byte> buffer)
+    {
+        Algorithm = EndianUtilities.ToInt32LittleEndian(buffer);
+        CompressionLevel = EndianUtilities.ToInt32LittleEndian(buffer[4..]);
+        return Size;
+    }
+
+    /// <inheritdoc />
+    public override void WriteTo(Span<byte> buffer)
+    {
+        EndianUtilities.WriteBytesLittleEndian(Algorithm, buffer);
+        EndianUtilities.WriteBytesLittleEndian(CompressionLevel, buffer[4..]);
+    }
+}
 
 /// <summary>
 /// Compression options for LZMA. (No options are supported).
@@ -322,4 +416,29 @@ public class LzmaCompressionOptions() : CompressionOptions(SquashFileSystemCompr
 /// <summary>
 /// ZStandard compression options. (No options are supported).
 /// </summary>
-public class ZStdCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.ZStd);
+/// <summary>
+/// Compression options for zstd: the compression level.
+/// </summary>
+public class ZStdCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.ZStd)
+{
+    /// <summary>
+    /// The zstd compression level (1-22).
+    /// </summary>
+    public int CompressionLevel { get; private set; } = 15;
+
+    /// <inheritdoc />
+    public override int Size => 4;
+
+    /// <inheritdoc />
+    public override int ReadFrom(ReadOnlySpan<byte> buffer)
+    {
+        CompressionLevel = EndianUtilities.ToInt32LittleEndian(buffer);
+        return Size;
+    }
+
+    /// <inheritdoc />
+    public override void WriteTo(Span<byte> buffer)
+    {
+        EndianUtilities.WriteBytesLittleEndian(CompressionLevel, buffer);
+    }
+}

Library/DiscUtils.SquashFs/ExtendedFileInode.cs

@@ -0,0 +1,73 @@
+//
+// Copyright (c) 2008-2011, Kenneth Bell
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense,
+// and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+//
+
+using System;
+using DiscUtils.Streams;
+
+namespace DiscUtils.SquashFs;
+
+/// <summary>
+/// The extended form of a regular file inode (type 9): 64-bit start block and size, the number of bytes that are
+/// sparse, the link count and an xattr index. mksquashfs uses it for any file larger than 4 GB, for sparse files
+/// and for files with more than one hard link, so a disk image or a hard-linked file inside a SquashFS image is one.
+/// The block list follows it exactly as for the basic form.
+/// </summary>
+internal sealed class ExtendedFileInode : RegularInode
+{
+    private long _fileSize;
+
+    public long SparseBytes;
+    public uint XattrIndex;
+
+    public override long FileSize
+    {
+        get => _fileSize;
+        set => _fileSize = value;
+    }
+
+    public override int Size => 56;
+
+    public override int ReadFrom(ReadOnlySpan<byte> buffer)
+    {
+        ReadHeader(buffer);
+        StartBlock = EndianUtilities.ToInt64LittleEndian(buffer[16..]);
+        _fileSize = EndianUtilities.ToInt64LittleEndian(buffer[24..]);
+        SparseBytes = EndianUtilities.ToInt64LittleEndian(buffer[32..]);
+        NumLinks = EndianUtilities.ToInt32LittleEndian(buffer[40..]);
+        FragmentKey = EndianUtilities.ToUInt32LittleEndian(buffer[44..]);
+        FragmentOffset = EndianUtilities.ToUInt32LittleEndian(buffer[48..]);
+        XattrIndex = EndianUtilities.ToUInt32LittleEndian(buffer[52..]);
+        return 56;
+    }
+
+    public override void WriteTo(Span<byte> buffer)
+    {
+        WriteHeader(buffer);
+        EndianUtilities.WriteBytesLittleEndian(StartBlock, buffer[16..]);
+        EndianUtilities.WriteBytesLittleEndian(_fileSize, buffer[24..]);
+        EndianUtilities.WriteBytesLittleEndian(SparseBytes, buffer[32..]);
+        EndianUtilities.WriteBytesLittleEndian(NumLinks, buffer[40..]);
+        EndianUtilities.WriteBytesLittleEndian(FragmentKey, buffer[44..]);
+        EndianUtilities.WriteBytesLittleEndian(FragmentOffset, buffer[48..]);
+        EndianUtilities.WriteBytesLittleEndian(XattrIndex, buffer[52..]);
+    }
+}

Library/DiscUtils.SquashFs/FileContentBuffer.cs

@@ -104,7 +104,16 @@ public override int Read(long pos, byte[] buffer, int offset, int count)
                 ++currentBlock;
             }
 
-            var blockOffset = (int)(pos % _context.SuperBlock.BlockSize);
+            var blockOffset = (int)(currentPos % _context.SuperBlock.BlockSize);
+            if ((_blockLengths[currentBlock] & 0x00FFFFFF) == 0)
+            {
+                // A sparse block: nothing stored, all zeros
+                var zeros = Math.Min(BlockLength(currentBlock) - blockOffset, totalToRead - totalRead);
+                Array.Clear(buffer, offset + totalRead, zeros);
+                totalRead += zeros;
+                currentPos += zeros;
+                continue;
+            }
 
             var block = _context.ReadBlock(currentBlockDiskStart, _blockLengths[currentBlock]);
 
@@ -117,6 +126,12 @@ public override int Read(long pos, byte[] buffer, int offset, int count)
         return totalRead;
     }
 
+    /// <summary>
+    /// The length of one block of the file: the block size, except for a last block that is not in a fragment.
+    /// </summary>
+    private int BlockLength(int block)
+        => (int)Math.Min(_context.SuperBlock.BlockSize, _inode.FileSize - (long)block * _context.SuperBlock.BlockSize);
+
     public override void Write(long pos, byte[] buffer, int offset, int count)
     {
         throw new NotSupportedException();
@@ -150,7 +165,16 @@ public override int Read(long pos, Span<byte> buffer)
                 ++currentBlock;
             }
 
-            var blockOffset = (int)(pos % _context.SuperBlock.BlockSize);
+            var blockOffset = (int)(currentPos % _context.SuperBlock.BlockSize);
+            if ((_blockLengths[currentBlock] & 0x00FFFFFF) == 0)
+            {
+                // A sparse block: nothing stored, all zeros
+                var zeros = Math.Min(BlockLength(currentBlock) - blockOffset, totalToRead - totalRead);
+                buffer.Slice(totalRead, zeros).Clear();
+                totalRead += zeros;
+                currentPos += zeros;
+                continue;
+            }
 
             var block = _context.ReadBlock(currentBlockDiskStart, _blockLengths[currentBlock]);
 

Library/DiscUtils.SquashFs/Inode.cs

@@ -45,7 +45,14 @@ public virtual long FileSize
 
     public abstract int Size { get; }
 
-    public virtual int ReadFrom(ReadOnlySpan<byte> buffer)
+    public virtual int ReadFrom(ReadOnlySpan<byte> buffer) => ReadHeader(buffer);
+
+    public virtual void WriteTo(Span<byte> buffer) => WriteHeader(buffer);
+
+    /// <summary>
+    /// Reads the 16-byte header every inode type starts with.
+    /// </summary>
+    protected int ReadHeader(ReadOnlySpan<byte> buffer)
     {
         Type = (InodeType)EndianUtilities.ToUInt16LittleEndian(buffer);
         Mode = EndianUtilities.ToUInt16LittleEndian(buffer[2..]);
@@ -56,7 +63,10 @@ public virtual int ReadFrom(ReadOnlySpan<byte> buffer)
         return 16;
     }
 
-    public virtual void WriteTo(Span<byte> buffer)
+    /// <summary>
+    /// Writes the 16-byte header every inode type starts with.
+    /// </summary>
+    protected void WriteHeader(Span<byte> buffer)
     {
         EndianUtilities.WriteBytesLittleEndian((ushort)Type, buffer);
         EndianUtilities.WriteBytesLittleEndian(Mode, buffer[2..]);
@@ -105,6 +115,7 @@ private static Inode InstantiateType(InodeType type)
             InodeType.Directory => new DirectoryInode(),
             InodeType.ExtendedDirectory => new ExtendedDirectoryInode(),
             InodeType.File => new RegularInode(),
+            InodeType.ExtendedFile => new ExtendedFileInode(),
             InodeType.Symlink => new SymlinkInode(),
             InodeType.CharacterDevice or InodeType.BlockDevice => new DeviceInode(),
             _ => throw new NotImplementedException($"Inode type not implemented: {type}"),

Library/DiscUtils.SquashFs/RegularInode.cs

@@ -30,7 +30,7 @@ internal class RegularInode : Inode
     private uint _fileSize;
     public uint FragmentKey;
     public uint FragmentOffset;
-    public uint StartBlock;
+    public long StartBlock;
 
     public override long FileSize
     {
@@ -67,7 +67,7 @@ public override void WriteTo(Span<byte> buffer)
     {
         base.WriteTo(buffer);
 
-        EndianUtilities.WriteBytesLittleEndian(StartBlock, buffer[16..]);
+        EndianUtilities.WriteBytesLittleEndian(checked((uint)StartBlock), buffer[16..]);
         EndianUtilities.WriteBytesLittleEndian(FragmentKey, buffer[20..]);
         EndianUtilities.WriteBytesLittleEndian(FragmentOffset, buffer[24..]);
         EndianUtilities.WriteBytesLittleEndian(_fileSize, buffer[28..]);

Tests/LibraryTests/LibraryTests.csproj

@@ -23,6 +23,8 @@
 
   <ItemGroup>
 	<EmbeddedResource Include="ExFat\Environment\exFAT.vhdx.gz" />
+	<EmbeddedResource Include="SquashFs\extended-inodes.sqsh" />
+	<EmbeddedResource Include="SquashFs\huge-sparse.sqsh" />
   </ItemGroup>
 
   <ItemGroup>