Add executable and object file signatures (#2)

* Add executable and object file signatures

Detect ELF, Mach-O (thin and universal, both byte orders), PE/COFF,
WebAssembly, ar, and zstd. PE follows e_lfanew within the 512-byte sniff
window. Universal Mach-O is gated on a plausible architecture count so
Java class files sharing the CA FE BA BE prefix are not misclassified.

Export the Format* constants so callers can switch on Result.Format
without string literals.

Closes #1

* Drop byte-swapped fat Mach-O signatures

Per mach-o/fat.h the fat header is always big-endian on disk; FAT_CIGAM
and FAT_CIGAM_64 are memory-order constants for LE readers, not
alternative on-disk byte sequences. Keep FAT_MAGIC and FAT_MAGIC_64
gated on the big-endian nfat_arch check and add a negative test for the
swapped prefix.

Author: andrew

README.md

@@ -49,15 +49,20 @@ mutable package state.
 
 `Kind` is `text`, `binary`, or `unknown`. `Format` and `MIME` describe the
 physical content. `Encoding` is set for accepted UTF-8, UTF-16LE, or UTF-16BE
-text and never appears as a MIME charset parameter.
+text and never appears as a MIME charset parameter. Compare `Format` against
+the exported `Format*` constants rather than string literals.
 
-The first format registry contains:
+The format registry contains:
 
-- ZIP, TAR, gzip, bzip2, xz, PDF, CFBF, PNG, JPEG, and GIF
+- ZIP, TAR, ar, gzip, bzip2, xz, zstd, PDF, CFBF, PNG, JPEG, and GIF
+- ELF, Mach-O (thin and universal), PE/COFF, and WebAssembly
 - plain text, HTML, XML, and SVG
 
 Detection uses bytes only. ZIP-based package types such as JAR, wheel, and
-NuGet remain `zip`, and compressed payloads are not opened. A caller can
+NuGet remain `zip`, and compressed payloads are not opened. A `CA FE BA BE`
+prefix is reported as Mach-O only when the following architecture count is
+plausible, so Java class files fall through unclassified. PE requires the
+`PE\0\0` signature to be reachable within the first 512 bytes. A caller can
 combine the result with filename or domain rules when it needs a semantic
 type.
 

magic.go

@@ -40,22 +40,31 @@ type Result struct {
 	NeedBytes int
 }
 
+// Format values reported in Result.Format.
 const (
-	formatText  = "text"
-	formatHTML  = "html"
-	formatXML   = "xml"
-	formatSVG   = "svg"
-	formatZIP   = "zip"
-	formatTAR   = "tar"
-	formatGZIP  = "gzip"
-	formatBZIP2 = "bzip2"
-	formatXZ    = "xz"
-	formatPDF   = "pdf"
-	formatCFBF  = "cfbf"
-	formatPNG   = "png"
-	formatJPEG  = "jpeg"
-	formatGIF   = "gif"
+	FormatText  = "text"
+	FormatHTML  = "html"
+	FormatXML   = "xml"
+	FormatSVG   = "svg"
+	FormatZIP   = "zip"
+	FormatTAR   = "tar"
+	FormatGZIP  = "gzip"
+	FormatBZIP2 = "bzip2"
+	FormatXZ    = "xz"
+	FormatZstd  = "zstd"
+	FormatPDF   = "pdf"
+	FormatCFBF  = "cfbf"
+	FormatPNG   = "png"
+	FormatJPEG  = "jpeg"
+	FormatGIF   = "gif"
+	FormatELF   = "elf"
+	FormatMachO = "mach-o"
+	FormatPE    = "pe"
+	FormatWASM  = "wasm"
+	FormatAR    = "ar"
+)
 
+const (
 	mimeText  = "text/plain"
 	mimeHTML  = "text/html"
 	mimeXML   = "text/xml"
@@ -70,6 +79,12 @@ const (
 	mimePNG   = "image/png"
 	mimeJPEG  = "image/jpeg"
 	mimeGIF   = "image/gif"
+	mimeZstd  = "application/zstd"
+	mimeELF   = "application/x-elf"
+	mimeMachO = "application/x-mach-binary"
+	mimePE    = "application/vnd.microsoft.portable-executable"
+	mimeWASM  = "application/wasm"
+	mimeAR    = "application/x-archive"
 
 	encodingUTF8    = "utf-8"
 	encodingUTF16LE = "utf-16le"
@@ -107,7 +122,7 @@ func detect(data []byte, prefix bool) Result {
 		result.Format = format
 		result.MIME = mime
 	} else if result.Kind == KindText {
-		result.Format = formatText
+		result.Format = FormatText
 		result.MIME = mimeText
 	}
 

magic_test.go

@@ -24,7 +24,7 @@ func TestDetectPrefix(t *testing.T) {
 			expect: Result{
 				Kind:     KindText,
 				MIME:     mimeText,
-				Format:   formatText,
+				Format:   FormatText,
 				Encoding: encodingUTF8,
 				Reason:   ReasonNeedMore,
 			},
@@ -43,7 +43,7 @@ func TestDetectPrefix(t *testing.T) {
 			expect: Result{
 				Kind:   KindBinary,
 				MIME:   mimePNG,
-				Format: formatPNG,
+				Format: FormatPNG,
 			},
 		},
 		{
@@ -124,7 +124,7 @@ func TestDetectDoesNotRetainInput(t *testing.T) {
 	assertResult(t, got, Result{
 		Kind:     KindText,
 		MIME:     mimeText,
-		Format:   formatText,
+		Format:   FormatText,
 		Encoding: encodingUTF8,
 	})
 }

signatures.go

@@ -7,6 +7,8 @@
 
 package magic
 
+import "encoding/binary"
+
 const (
 	// sniffLength is the furthest byte inspected by any signature. A binary
 	// rule that reads beyond it must also update prefixResultCanChange.
@@ -17,6 +19,20 @@ const (
 	tarChecksumTo   = 156
 	xmlCloseLength  = 2
 	octalBase       = 8
+
+	// peHeaderOffsetAt is the location of the uint32le e_lfanew field in the
+	// DOS header, which holds the offset of the "PE\0\0" signature.
+	peHeaderOffsetAt = 0x3c
+	peSignatureLen   = 4
+
+	// machOFatArchLimit separates a Mach-O universal binary from a Java
+	// class file, which share the CA FE BA BE prefix. Bytes 4-7 are the
+	// big-endian architecture count in a fat header and (minor||major)
+	// version in a class file; the class-file major version has been at
+	// least 45 since JDK 1.0.2 while no fat binary approaches that many
+	// architectures.
+	machOFatHeaderLen = 8
+	machOFatArchLimit = 40
 )
 
 var htmlSignatures = [...]string{
@@ -43,46 +59,91 @@ func binaryFormat(data []byte) (format, mime string) {
 	case hasPrefix(data, "PK\x03\x04"),
 		hasPrefix(data, "PK\x05\x06"),
 		hasPrefix(data, "PK\x07\x08"):
-		return formatZIP, mimeZIP
+		return FormatZIP, mimeZIP
 	case hasPrefix(data, "\x1f\x8b\x08"):
-		return formatGZIP, mimeGZIP
+		return FormatGZIP, mimeGZIP
 	case len(data) >= 4 &&
 		hasPrefix(data, "BZh") &&
 		data[3] >= '1' && data[3] <= '9':
-		return formatBZIP2, mimeBZIP2
+		return FormatBZIP2, mimeBZIP2
 	case hasPrefix(data, "\xfd7zXZ\x00"):
-		return formatXZ, mimeXZ
+		return FormatXZ, mimeXZ
 	case hasPrefix(data, "%PDF-"):
-		return formatPDF, mimePDF
+		return FormatPDF, mimePDF
 	case hasPrefix(data, "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
-		return formatCFBF, mimeCFBF
+		return FormatCFBF, mimeCFBF
 	case hasPrefix(data, "\x89PNG\r\n\x1a\n"):
-		return formatPNG, mimePNG
+		return FormatPNG, mimePNG
 	case hasPrefix(data, "\xff\xd8\xff"):
-		return formatJPEG, mimeJPEG
+		return FormatJPEG, mimeJPEG
 	case hasPrefix(data, "GIF87a"), hasPrefix(data, "GIF89a"):
-		return formatGIF, mimeGIF
+		return FormatGIF, mimeGIF
+	case hasPrefix(data, "\x28\xb5\x2f\xfd"):
+		return FormatZstd, mimeZstd
+	case hasPrefix(data, "\x7fELF"):
+		return FormatELF, mimeELF
+	case hasPrefix(data, "\xcf\xfa\xed\xfe"),
+		hasPrefix(data, "\xce\xfa\xed\xfe"),
+		hasPrefix(data, "\xfe\xed\xfa\xcf"),
+		hasPrefix(data, "\xfe\xed\xfa\xce"):
+		return FormatMachO, mimeMachO
+	case machOFatHeader(data):
+		return FormatMachO, mimeMachO
+	case hasPrefix(data, "\x00asm"):
+		return FormatWASM, mimeWASM
+	case hasPrefix(data, "!<arch>\n"):
+		return FormatAR, mimeAR
+	case peHeader(data):
+		return FormatPE, mimePE
 	case validTARHeader(data):
-		return formatTAR, mimeTAR
+		return FormatTAR, mimeTAR
 	default:
 		return "", ""
 	}
 }
 
+func machOFatHeader(data []byte) bool {
+	if len(data) < machOFatHeaderLen {
+		return false
+	}
+	// Fat headers are always big-endian on disk per mach-o/fat.h; FAT_CIGAM
+	// is a memory-order constant, not an alternative on-disk signature.
+	if !hasPrefix(data, "\xca\xfe\xba\xbe") &&
+		!hasPrefix(data, "\xca\xfe\xba\xbf") {
+		return false
+	}
+	nfat := binary.BigEndian.Uint32(data[4:machOFatHeaderLen])
+	return nfat > 0 && nfat < machOFatArchLimit
+}
+
+func peHeader(data []byte) bool {
+	if !hasPrefix(data, "MZ") || len(data) < peHeaderOffsetAt+4 {
+		return false
+	}
+	offset := binary.LittleEndian.Uint32(data[peHeaderOffsetAt:])
+	// Bound to sniffLength so prefixResultCanChange stays correct. PE files
+	// with a DOS stub larger than the sniff window are not recognised.
+	if offset < peHeaderOffsetAt+4 || offset > sniffLength-peSignatureLen ||
+		int(offset)+peSignatureLen > len(data) {
+		return false
+	}
+	return hasPrefix(data[offset:], "PE\x00\x00")
+}
+
 func textFormat(data []byte) (format, mime string) {
 	if len(data) > sniffLength {
 		data = data[:sniffLength]
 	}
 
 	first := skipWhitespace(data, 0)
 	if isSVG(data, first) {
-		return formatSVG, mimeSVG
+		return FormatSVG, mimeSVG
 	}
 	if hasPrefix(data[first:], "<?xml") {
-		return formatXML, mimeXML
+		return FormatXML, mimeXML
 	}
 	if isHTML(data, first) {
-		return formatHTML, mimeHTML
+		return FormatHTML, mimeHTML
 	}
 	return "", ""
 }