diff --git a/src/BinaryParsers/ElfBinary/ElfBinary.cs b/src/BinaryParsers/ElfBinary/ElfBinary.cs index f38c14fda..abc7bbd42 100644 --- a/src/BinaryParsers/ElfBinary/ElfBinary.cs +++ b/src/BinaryParsers/ElfBinary/ElfBinary.cs @@ -2,11 +2,14 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Text; +using ELFSharp; using ELFSharp.ELF; using ELFSharp.ELF.Sections; using ELFSharp.ELF.Segments; @@ -21,6 +24,11 @@ namespace Microsoft.CodeAnalysis.BinaryParsers /// public class ElfBinary : BinaryBase, IDwarfBinary { + // ELF section flag bit for SHF_COMPRESSED. + private const ulong CompressedSectionFlag = 0x800UL; + private const uint ZlibCompressionType = 1; + private const uint ZstdCompressionType = 2; + public ElfBinary(Uri uri, string localSymbolDirectories = null, bool forceComprehensiveParsing = false, @@ -393,7 +401,7 @@ private byte[] LoadSection(string sectionName) { if (section.Name == sectionName + ".dwo" || section.Name == sectionName) { - return section.GetContents(); + return GetSectionContents(section, Is64bit, ELF.Endianess); } } @@ -407,7 +415,8 @@ private IDwarfStringReader CreateDebugStringsReader() .FirstOrDefault(candidate => candidate.Name == SectionName.DebugStr || candidate.Name == SectionName.DebugStr + ".dwo"); - if (section != null && ShouldUseFileBackedDwarfStringReader( + // File offsets cannot address strings inside a compressed section. + if (section != null && !IsCompressedSection(section) && ShouldUseFileBackedDwarfStringReader( section.Size, this.dwarfStringSectionFileReadThreshold)) { @@ -424,6 +433,86 @@ internal static bool ShouldUseFileBackedDwarfStringReader( return fileReadThreshold.HasValue && sectionSize >= fileReadThreshold.Value; } + internal static byte[] GetSectionContents(ISection section, bool is64bit, Endianess endianess) + { + byte[] contents = section.GetContents(); + + return IsCompressedSection(section) + ? DecompressSectionContents(contents, is64bit, endianess) + : contents; + } + + internal static byte[] DecompressSectionContents(byte[] contents, bool is64bit, Endianess endianess) + { + if (contents.Length == 0) + { + return contents; + } + + int headerSize = is64bit ? 24 : 12; + + if (contents.Length < headerSize) + { + throw new InvalidOperationException("Compressed ELF section header is truncated."); + } + + uint compressionType = ReadUInt32(contents.AsSpan(0, sizeof(uint)), endianess); + + if (compressionType == ZstdCompressionType) + { + throw new NotSupportedException("ELF zstd-compressed sections are not supported on this runtime target because there is no built-in Zstandard stream to decode them."); + } + + if (compressionType != ZlibCompressionType) + { + throw new NotSupportedException($"Unsupported ELF compression type: {compressionType}."); + } + + ulong uncompressedSize = is64bit + ? ReadUInt64(contents.AsSpan(8, sizeof(ulong)), endianess) + : ReadUInt32(contents.AsSpan(4, sizeof(uint)), endianess); + + if (uncompressedSize > (ulong)Array.MaxLength) + { + throw new InvalidDataException("Uncompressed ELF section size exceeds the maximum array length."); + } + + using var compressedStream = new MemoryStream(contents, headerSize, contents.Length - headerSize, writable: false); + using var zlibStream = new ZLibStream(compressedStream, CompressionMode.Decompress); + using var decompressedStream = new MemoryStream((int)uncompressedSize); + + zlibStream.CopyTo(decompressedStream); + + byte[] decompressedContents = decompressedStream.ToArray(); + + if ((ulong)decompressedContents.LongLength != uncompressedSize) + { + throw new InvalidOperationException("Compressed ELF section size does not match the ELF header."); + } + + return decompressedContents; + } + + private static uint ReadUInt32(ReadOnlySpan contents, Endianess endianess) + { + return endianess == Endianess.LittleEndian + ? BinaryPrimitives.ReadUInt32LittleEndian(contents) + : BinaryPrimitives.ReadUInt32BigEndian(contents); + } + + private static ulong ReadUInt64(ReadOnlySpan contents, Endianess endianess) + { + return endianess == Endianess.LittleEndian + ? BinaryPrimitives.ReadUInt64LittleEndian(contents) + : BinaryPrimitives.ReadUInt64BigEndian(contents); + } + + private static bool IsCompressedSection(ISection section) + { + return section is Section elfSection + && (elfSection.RawFlags & CompressedSectionFlag) != 0; + } + /// /// Gets the section address after loading into memory. /// diff --git a/src/Test.UnitTests.BinaryParsers/Elf/ElfBinaryTests.cs b/src/Test.UnitTests.BinaryParsers/Elf/ElfBinaryTests.cs index 59d3e3fe9..62ccca04e 100644 --- a/src/Test.UnitTests.BinaryParsers/Elf/ElfBinaryTests.cs +++ b/src/Test.UnitTests.BinaryParsers/Elf/ElfBinaryTests.cs @@ -2,10 +2,16 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Reflection; +using System.Text; + +using ELFSharp; +using ELFSharp.ELF.Sections; using FluentAssertions; @@ -192,6 +198,45 @@ public void ValidateDwarfV5_WithO2() binary.GetLanguage().Should().Be(DwarfLanguage.C11); } + [Theory] + [InlineData("hello-dwarf4-o2-compressed", 4, DwarfLanguage.C99, null)] + [InlineData("hello-dwarf4-o2-compressed", 4, DwarfLanguage.C99, 0UL)] + [InlineData("hello-dwarf5-o2-compressed", 5, DwarfLanguage.C11, null)] + [InlineData("hello-dwarf5-o2-compressed", 5, DwarfLanguage.C11, 0UL)] + public void ValidateDwarf_WithCompressedSections( + string fileName, + int expectedVersion, + DwarfLanguage expectedLanguage, + ulong? fileReadThreshold) + { + // Compressed copies of the existing ELF fixtures, generated with GNU binutils 2.42: + // x86_64-linux-gnu-objcopy --compress-debug-sections=zlib-gabi hello-dwarf4-o2 hello-dwarf4-o2-compressed + // x86_64-linux-gnu-objcopy --compress-debug-sections=zlib-gabi hello-dwarf5-o2 hello-dwarf5-o2-compressed + string filePath = Path.Combine(TestData, "Dwarf", fileName); + using var binary = new ElfBinary(new Uri(filePath), dwarfStringSectionFileReadThreshold: fileReadThreshold); + string originalFilePath = Path.Combine(TestData, "Dwarf", $"hello-dwarf{expectedVersion}-o2"); + using var originalBinary = new ElfBinary(new Uri(originalFilePath)); + + binary.LoadException.Should().BeNull(); + binary.Valid.Should().BeTrue(); + binary.ELF.Sections + .OfType>() + .Where(section => section.Name == ".debug_info" || section.Name == ".debug_str") + .Should().HaveCount(2) + .And.OnlyContain(section => (section.RawFlags & 0x800UL) != 0); // SHF_COMPRESSED + + // A zero threshold must still decode compressed strings instead of reading raw file offsets. + binary.DwarfVersion.Should().Be(expectedVersion); + binary.DebugFileType.Should().Be(DebugFileType.DebugIncluded); + binary.DebugFileLoaded.Should().BeTrue(); + binary.GetLanguage().Should().Be(expectedLanguage); + binary.CommandLineInfos + .Where(info => info.Language != DwarfLanguage.Unknown) + .Should().NotBeEmpty() + .And.OnlyContain(info => info.CommandLine.Contains("O2")); + binary.DebugLine.Should().NotBeEmpty().And.Equal(originalBinary.DebugLine); + } + [Fact] public void ValidateDwarfV5_Rust() { @@ -203,6 +248,129 @@ public void ValidateDwarfV5_Rust() binary.GetLanguage().Should().Be(DwarfLanguage.Rust); } + [Theory] + [InlineData(Endianess.LittleEndian)] + [InlineData(Endianess.BigEndian)] + public void DecompressSectionContents_Elf64CompressedSection_Decompresses(Endianess endianess) + { + byte[] originalContents = Encoding.ASCII.GetBytes("dwarf-v5-go-section"); + + byte[] compressedContents; + using (var compressedStream = new MemoryStream()) + { + using (var zlibStream = new ZLibStream(compressedStream, CompressionLevel.SmallestSize, leaveOpen: true)) + { + zlibStream.Write(originalContents, 0, originalContents.Length); + } + + compressedContents = compressedStream.ToArray(); + } + + byte[] sectionContents = CreateCompressedSectionContents(compressedContents, (ulong)originalContents.Length, is64bit: true, endianess: endianess); + + byte[] decompressedContents = ElfBinary.DecompressSectionContents(sectionContents, is64bit: true, endianess: endianess); + + decompressedContents.Should().Equal(originalContents); + } + + [Theory] + [InlineData(Endianess.LittleEndian)] + [InlineData(Endianess.BigEndian)] + public void DecompressSectionContents_Elf32CompressedSection_Decompresses(Endianess endianess) + { + byte[] originalContents = Encoding.ASCII.GetBytes("dwarf-v5-go-section"); + + byte[] compressedContents; + using (var compressedStream = new MemoryStream()) + { + using (var zlibStream = new ZLibStream(compressedStream, CompressionLevel.SmallestSize, leaveOpen: true)) + { + zlibStream.Write(originalContents, 0, originalContents.Length); + } + + compressedContents = compressedStream.ToArray(); + } + + byte[] sectionContents = CreateCompressedSectionContents(compressedContents, (ulong)originalContents.Length, is64bit: false, endianess: endianess); + + byte[] decompressedContents = ElfBinary.DecompressSectionContents(sectionContents, is64bit: false, endianess: endianess); + + decompressedContents.Should().Equal(originalContents); + } + + [Theory] + [InlineData(false, Endianess.LittleEndian, false)] + [InlineData(false, Endianess.BigEndian, false)] + [InlineData(true, Endianess.LittleEndian, false)] + [InlineData(true, Endianess.BigEndian, false)] + [InlineData(true, Endianess.LittleEndian, true)] + [InlineData(true, Endianess.BigEndian, true)] + public void DecompressSectionContents_UncompressedSizeExceedsArrayMaxLength_Throws(bool is64bit, Endianess endianess, bool sizeExceeds32Bits) + { + byte[] originalContents = Encoding.ASCII.GetBytes("dwarf-v5-go-section"); + + byte[] compressedContents; + using (var compressedStream = new MemoryStream()) + { + using (var zlibStream = new ZLibStream(compressedStream, CompressionLevel.SmallestSize, leaveOpen: true)) + { + zlibStream.Write(originalContents, 0, originalContents.Length); + } + + compressedContents = compressedStream.ToArray(); + } + + // Keep the low 32 bits equal to the payload length to catch truncated ELF64 size reads. + ulong uncompressedSize = sizeExceeds32Bits + ? (1UL << 32) + (ulong)originalContents.Length + : (ulong)Array.MaxLength + 1; + byte[] sectionContents = CreateCompressedSectionContents(compressedContents, uncompressedSize, is64bit, endianess); + + Action decompress = () => ElfBinary.DecompressSectionContents(sectionContents, is64bit, endianess); + + decompress.Should().Throw(); + } + + private static byte[] CreateCompressedSectionContents(byte[] compressedContents, ulong uncompressedSize, bool is64bit, Endianess endianess) + { + int headerSize = is64bit ? 24 : 12; + byte[] contents = new byte[headerSize + compressedContents.Length]; + + if (endianess == Endianess.LittleEndian) + { + BinaryPrimitives.WriteUInt32LittleEndian(contents.AsSpan(0, sizeof(uint)), 1); + + if (is64bit) + { + BinaryPrimitives.WriteUInt64LittleEndian(contents.AsSpan(8, sizeof(ulong)), uncompressedSize); + BinaryPrimitives.WriteUInt64LittleEndian(contents.AsSpan(16, sizeof(ulong)), 1); + } + else + { + BinaryPrimitives.WriteUInt32LittleEndian(contents.AsSpan(4, sizeof(uint)), checked((uint)uncompressedSize)); + BinaryPrimitives.WriteUInt32LittleEndian(contents.AsSpan(8, sizeof(uint)), 1); + } + } + else + { + BinaryPrimitives.WriteUInt32BigEndian(contents.AsSpan(0, sizeof(uint)), 1); + + if (is64bit) + { + BinaryPrimitives.WriteUInt64BigEndian(contents.AsSpan(8, sizeof(ulong)), uncompressedSize); + BinaryPrimitives.WriteUInt64BigEndian(contents.AsSpan(16, sizeof(ulong)), 1); + } + else + { + BinaryPrimitives.WriteUInt32BigEndian(contents.AsSpan(4, sizeof(uint)), checked((uint)uncompressedSize)); + BinaryPrimitives.WriteUInt32BigEndian(contents.AsSpan(8, sizeof(uint)), 1); + } + } + + compressedContents.CopyTo(contents, headerSize); + return contents; + } + [Fact] public void ValidateDwarfV4_WithO2_Split_DebugFileExists() { diff --git a/src/Test.UnitTests.BinaryParsers/TestData/Dwarf/hello-dwarf4-o2-compressed b/src/Test.UnitTests.BinaryParsers/TestData/Dwarf/hello-dwarf4-o2-compressed new file mode 100644 index 000000000..9430e1c5b Binary files /dev/null and b/src/Test.UnitTests.BinaryParsers/TestData/Dwarf/hello-dwarf4-o2-compressed differ diff --git a/src/Test.UnitTests.BinaryParsers/TestData/Dwarf/hello-dwarf5-o2-compressed b/src/Test.UnitTests.BinaryParsers/TestData/Dwarf/hello-dwarf5-o2-compressed new file mode 100644 index 000000000..41bf57536 Binary files /dev/null and b/src/Test.UnitTests.BinaryParsers/TestData/Dwarf/hello-dwarf5-o2-compressed differ