Handle compressed DWARF sections in ELF binaries

src/BinaryParsers/ElfBinary/ElfBinary.cs

@@ -2,11 +2,14 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Buffers.Binary;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using System.Text;
 
+using ELFSharp;
 using ELFSharp.ELF;
 using ELFSharp.ELF.Sections;
 using ELFSharp.ELF.Segments;
@@ -21,6 +24,11 @@ namespace Microsoft.CodeAnalysis.BinaryParsers
     /// </summary>
     public class ElfBinary : BinaryBase, IDwarfBinary
     {
+        // ELF section flag bit for SHF_COMPRESSED.
+        private const ulong CompressedSectionFlag = 0x800UL;
+        private const uint ZlibCompressionType = 1;
+        private const uint ZstdCompressionType = 2;
+
         public ElfBinary(Uri uri,
                          string localSymbolDirectories = null,
                          bool forceComprehensiveParsing = false,
@@ -393,7 +401,7 @@ private byte[] LoadSection(string sectionName)
             {
                 if (section.Name == sectionName + ".dwo" || section.Name == sectionName)
                 {
-                    return section.GetContents();
+                    return GetSectionContents(section, Is64bit, ELF.Endianess);
                 }
             }
 
@@ -407,7 +415,8 @@ private IDwarfStringReader CreateDebugStringsReader()
                 .FirstOrDefault(candidate => candidate.Name == SectionName.DebugStr ||
                                              candidate.Name == SectionName.DebugStr + ".dwo");
 
-            if (section != null && ShouldUseFileBackedDwarfStringReader(
+            // File offsets cannot address strings inside a compressed section.
+            if (section != null && !IsCompressedSection(section) && ShouldUseFileBackedDwarfStringReader(
                 section.Size,
                 this.dwarfStringSectionFileReadThreshold))
             {
@@ -424,6 +433,86 @@ internal static bool ShouldUseFileBackedDwarfStringReader(
             return fileReadThreshold.HasValue && sectionSize >= fileReadThreshold.Value;
         }
 
+        internal static byte[] GetSectionContents(ISection section, bool is64bit, Endianess endianess)
+        {
+            byte[] contents = section.GetContents();
+
+            return IsCompressedSection(section)
+                ? DecompressSectionContents(contents, is64bit, endianess)
+                : contents;
+        }
+
+        internal static byte[] DecompressSectionContents(byte[] contents, bool is64bit, Endianess endianess)
+        {
+            if (contents.Length == 0)
+            {
+                return contents;
+            }
+
+            int headerSize = is64bit ? 24 : 12;
+
+            if (contents.Length < headerSize)
+            {
+                throw new InvalidOperationException("Compressed ELF section header is truncated.");
+            }
+
+            uint compressionType = ReadUInt32(contents.AsSpan(0, sizeof(uint)), endianess);
+
+            if (compressionType == ZstdCompressionType)
+            {
+                throw new NotSupportedException("ELF zstd-compressed sections are not supported on this runtime target because there is no built-in Zstandard stream to decode them.");
+            }
+
+            if (compressionType != ZlibCompressionType)
+            {
+                throw new NotSupportedException($"Unsupported ELF compression type: {compressionType}.");
+            }
+
+            ulong uncompressedSize = is64bit
+                ? ReadUInt64(contents.AsSpan(8, sizeof(ulong)), endianess)
+                : ReadUInt32(contents.AsSpan(4, sizeof(uint)), endianess);
+
+            if (uncompressedSize > (ulong)Array.MaxLength)
+            {
+                throw new InvalidDataException("Uncompressed ELF section size exceeds the maximum array length.");
+            }
+
+            using var compressedStream = new MemoryStream(contents, headerSize, contents.Length - headerSize, writable: false);
+            using var zlibStream = new ZLibStream(compressedStream, CompressionMode.Decompress);
+            using var decompressedStream = new MemoryStream((int)uncompressedSize);
+
+            zlibStream.CopyTo(decompressedStream);
+
+            byte[] decompressedContents = decompressedStream.ToArray();
+
+            if ((ulong)decompressedContents.LongLength != uncompressedSize)
+            {
+                throw new InvalidOperationException("Compressed ELF section size does not match the ELF header.");
+            }
+
+            return decompressedContents;
+        }
+
+        private static uint ReadUInt32(ReadOnlySpan<byte> contents, Endianess endianess)
+        {
+            return endianess == Endianess.LittleEndian
+                ? BinaryPrimitives.ReadUInt32LittleEndian(contents)
+                : BinaryPrimitives.ReadUInt32BigEndian(contents);
+        }
+
+        private static ulong ReadUInt64(ReadOnlySpan<byte> contents, Endianess endianess)
+        {
+            return endianess == Endianess.LittleEndian
+                ? BinaryPrimitives.ReadUInt64LittleEndian(contents)
+                : BinaryPrimitives.ReadUInt64BigEndian(contents);
+        }
+
+        private static bool IsCompressedSection(ISection section)
+        {
+            return section is Section<ulong> elfSection
+                && (elfSection.RawFlags & CompressedSectionFlag) != 0;
+        }
+
         /// <summary>
         /// Gets the section address after loading into memory.
         /// </summary>

src/Test.UnitTests.BinaryParsers/Elf/ElfBinaryTests.cs

@@ -2,10 +2,16 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Buffers.Binary;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using System.Reflection;
+using System.Text;
+
+using ELFSharp;
+using ELFSharp.ELF.Sections;
 
 using FluentAssertions;
 
@@ -192,6 +198,45 @@ public void ValidateDwarfV5_WithO2()
             binary.GetLanguage().Should().Be(DwarfLanguage.C11);
         }
 
+        [Theory]
+        [InlineData("hello-dwarf4-o2-compressed", 4, DwarfLanguage.C99, null)]
+        [InlineData("hello-dwarf4-o2-compressed", 4, DwarfLanguage.C99, 0UL)]
+        [InlineData("hello-dwarf5-o2-compressed", 5, DwarfLanguage.C11, null)]
+        [InlineData("hello-dwarf5-o2-compressed", 5, DwarfLanguage.C11, 0UL)]
+        public void ValidateDwarf_WithCompressedSections(
+            string fileName,
+            int expectedVersion,
+            DwarfLanguage expectedLanguage,
+            ulong? fileReadThreshold)
+        {
+            // Compressed copies of the existing ELF fixtures, generated with GNU binutils 2.42:
+            // x86_64-linux-gnu-objcopy --compress-debug-sections=zlib-gabi hello-dwarf4-o2 hello-dwarf4-o2-compressed
+            // x86_64-linux-gnu-objcopy --compress-debug-sections=zlib-gabi hello-dwarf5-o2 hello-dwarf5-o2-compressed
+            string filePath = Path.Combine(TestData, "Dwarf", fileName);
+            using var binary = new ElfBinary(new Uri(filePath), dwarfStringSectionFileReadThreshold: fileReadThreshold);
+            string originalFilePath = Path.Combine(TestData, "Dwarf", $"hello-dwarf{expectedVersion}-o2");
+            using var originalBinary = new ElfBinary(new Uri(originalFilePath));
+
+            binary.LoadException.Should().BeNull();
+            binary.Valid.Should().BeTrue();
+            binary.ELF.Sections
+                .OfType<Section<ulong>>()
+                .Where(section => section.Name == ".debug_info" || section.Name == ".debug_str")
+                .Should().HaveCount(2)
+                .And.OnlyContain(section => (section.RawFlags & 0x800UL) != 0); // SHF_COMPRESSED
+
+            // A zero threshold must still decode compressed strings instead of reading raw file offsets.
+            binary.DwarfVersion.Should().Be(expectedVersion);
+            binary.DebugFileType.Should().Be(DebugFileType.DebugIncluded);
+            binary.DebugFileLoaded.Should().BeTrue();
+            binary.GetLanguage().Should().Be(expectedLanguage);
+            binary.CommandLineInfos
+                .Where(info => info.Language != DwarfLanguage.Unknown)
+                .Should().NotBeEmpty()
+                .And.OnlyContain(info => info.CommandLine.Contains("O2"));
+            binary.DebugLine.Should().NotBeEmpty().And.Equal(originalBinary.DebugLine);
+        }
+
         [Fact]
         public void ValidateDwarfV5_Rust()
         {
@@ -203,6 +248,129 @@ public void ValidateDwarfV5_Rust()
             binary.GetLanguage().Should().Be(DwarfLanguage.Rust);
         }
 
+        [Theory]
+        [InlineData(Endianess.LittleEndian)]
+        [InlineData(Endianess.BigEndian)]
+        public void DecompressSectionContents_Elf64CompressedSection_Decompresses(Endianess endianess)
+        {
+            byte[] originalContents = Encoding.ASCII.GetBytes("dwarf-v5-go-section");
+
+            byte[] compressedContents;
+            using (var compressedStream = new MemoryStream())
+            {
+                using (var zlibStream = new ZLibStream(compressedStream, CompressionLevel.SmallestSize, leaveOpen: true))
+                {
+                    zlibStream.Write(originalContents, 0, originalContents.Length);
+                }
+
+                compressedContents = compressedStream.ToArray();
+            }
+
+            byte[] sectionContents = CreateCompressedSectionContents(compressedContents, (ulong)originalContents.Length, is64bit: true, endianess: endianess);
+
+            byte[] decompressedContents = ElfBinary.DecompressSectionContents(sectionContents, is64bit: true, endianess: endianess);
+
+            decompressedContents.Should().Equal(originalContents);
+        }
+
+        [Theory]
+        [InlineData(Endianess.LittleEndian)]
+        [InlineData(Endianess.BigEndian)]
+        public void DecompressSectionContents_Elf32CompressedSection_Decompresses(Endianess endianess)
+        {
+            byte[] originalContents = Encoding.ASCII.GetBytes("dwarf-v5-go-section");
+
+            byte[] compressedContents;
+            using (var compressedStream = new MemoryStream())
+            {
+                using (var zlibStream = new ZLibStream(compressedStream, CompressionLevel.SmallestSize, leaveOpen: true))
+                {
+                    zlibStream.Write(originalContents, 0, originalContents.Length);
+                }
+
+                compressedContents = compressedStream.ToArray();
+            }
+
+            byte[] sectionContents = CreateCompressedSectionContents(compressedContents, (ulong)originalContents.Length, is64bit: false, endianess: endianess);
+
+            byte[] decompressedContents = ElfBinary.DecompressSectionContents(sectionContents, is64bit: false, endianess: endianess);
+
+            decompressedContents.Should().Equal(originalContents);
+        }
+
+        [Theory]
+        [InlineData(false, Endianess.LittleEndian, false)]
+        [InlineData(false, Endianess.BigEndian, false)]
+        [InlineData(true, Endianess.LittleEndian, false)]
+        [InlineData(true, Endianess.BigEndian, false)]
+        [InlineData(true, Endianess.LittleEndian, true)]
+        [InlineData(true, Endianess.BigEndian, true)]
+        public void DecompressSectionContents_UncompressedSizeExceedsArrayMaxLength_Throws(bool is64bit, Endianess endianess, bool sizeExceeds32Bits)
+        {
+            byte[] originalContents = Encoding.ASCII.GetBytes("dwarf-v5-go-section");
+
+            byte[] compressedContents;
+            using (var compressedStream = new MemoryStream())
+            {
+                using (var zlibStream = new ZLibStream(compressedStream, CompressionLevel.SmallestSize, leaveOpen: true))
+                {
+                    zlibStream.Write(originalContents, 0, originalContents.Length);
+                }
+
+                compressedContents = compressedStream.ToArray();
+            }
+
+            // Keep the low 32 bits equal to the payload length to catch truncated ELF64 size reads.
+            ulong uncompressedSize = sizeExceeds32Bits
+                ? (1UL << 32) + (ulong)originalContents.Length
+                : (ulong)Array.MaxLength + 1;
+            byte[] sectionContents = CreateCompressedSectionContents(compressedContents, uncompressedSize, is64bit, endianess);
+
+            Action decompress = () => ElfBinary.DecompressSectionContents(sectionContents, is64bit, endianess);
+
+            decompress.Should().Throw<InvalidDataException>();
+        }
+
+        private static byte[] CreateCompressedSectionContents(byte[] compressedContents, ulong uncompressedSize, bool is64bit, Endianess endianess)
+        {
+            int headerSize = is64bit ? 24 : 12;
+            byte[] contents = new byte[headerSize + compressedContents.Length];
+
+            if (endianess == Endianess.LittleEndian)
+            {
+                BinaryPrimitives.WriteUInt32LittleEndian(contents.AsSpan(0, sizeof(uint)), 1);
+
+                if (is64bit)
+                {
+                    BinaryPrimitives.WriteUInt64LittleEndian(contents.AsSpan(8, sizeof(ulong)), uncompressedSize);
+                    BinaryPrimitives.WriteUInt64LittleEndian(contents.AsSpan(16, sizeof(ulong)), 1);
+                }
+                else
+                {
+                    BinaryPrimitives.WriteUInt32LittleEndian(contents.AsSpan(4, sizeof(uint)), checked((uint)uncompressedSize));
+                    BinaryPrimitives.WriteUInt32LittleEndian(contents.AsSpan(8, sizeof(uint)), 1);
+                }
+            }
+            else
+            {
+                BinaryPrimitives.WriteUInt32BigEndian(contents.AsSpan(0, sizeof(uint)), 1);
+
+                if (is64bit)
+                {
+                    BinaryPrimitives.WriteUInt64BigEndian(contents.AsSpan(8, sizeof(ulong)), uncompressedSize);
+                    BinaryPrimitives.WriteUInt64BigEndian(contents.AsSpan(16, sizeof(ulong)), 1);
+                }
+                else
+                {
+                    BinaryPrimitives.WriteUInt32BigEndian(contents.AsSpan(4, sizeof(uint)), checked((uint)uncompressedSize));
+                    BinaryPrimitives.WriteUInt32BigEndian(contents.AsSpan(8, sizeof(uint)), 1);
+                }
+            }
+
+            compressedContents.CopyTo(contents, headerSize);
+            return contents;
+        }
+
         [Fact]
         public void ValidateDwarfV4_WithO2_Split_DebugFileExists()
         {