Detect native PHAR archives (#7)

Author: andrew

README.md

@@ -54,7 +54,7 @@ the exported `Format*` constants rather than string literals.
 
 The format registry contains:
 
-- ZIP, TAR, ar, gzip, bzip2, xz, zstd, PDF, CFBF, PNG, JPEG, and GIF
+- ZIP, TAR, native PHAR, ar, gzip, bzip2, xz, zstd, PDF, CFBF, PNG, JPEG, and GIF
 - ELF, Mach-O (thin and universal), PE/COFF, and WebAssembly
 - plain text, HTML, XML, and SVG
 
@@ -89,9 +89,10 @@ Run the package benchmarks on the target machine:
 go test -run '^$' -bench . -benchmem
 ```
 
-The implementation scans at most 512 bytes for registered signatures. Text
-validation is linear in the supplied byte count and uses fixed auxiliary
-memory.
+Fixed signatures inspect at most 512 bytes, while native PHAR detection
+searches for the end of the PHP stub and validates the manifest and stored
+payload bounds. Text validation is linear in the supplied byte count and uses
+fixed auxiliary memory.
 
 ## Provenance
 

fuzz_test.go

@@ -31,6 +31,7 @@ func FuzzDetect(f *testing.F) {
 		f.Add(seed)
 	}
 	f.Add(makeTAR(f))
+	f.Add(makeNativePHAR(pharTestStub, "", nil, pharTestEntry{name: "file", content: []byte("data")}))
 
 	f.Fuzz(func(t *testing.T, data []byte) {
 		first := Detect(data)

magic.go

@@ -48,6 +48,7 @@ const (
 	FormatSVG   = "svg"
 	FormatZIP   = "zip"
 	FormatTAR   = "tar"
+	FormatPHAR  = "phar"
 	FormatGZIP  = "gzip"
 	FormatBZIP2 = "bzip2"
 	FormatXZ    = "xz"
@@ -71,6 +72,7 @@ const (
 	mimeSVG   = "image/svg+xml"
 	mimeZIP   = "application/zip"
 	mimeTAR   = "application/x-tar"
+	mimePHAR  = "application/x-phar"
 	mimeGZIP  = "application/gzip"
 	mimeBZIP2 = "application/x-bzip2"
 	mimeXZ    = "application/x-xz"
@@ -108,15 +110,16 @@ func DetectPrefix(prefix []byte) Result {
 }
 
 func detect(data []byte, prefix bool) Result {
-	if format, mime := binaryFormat(data); format != "" {
+	format, mime, binaryNeedsMore := binaryFormatState(data)
+	if format != "" {
 		return Result{
 			Kind:   KindBinary,
 			MIME:   mime,
 			Format: format,
 		}
 	}
 
-	format, mime := textFormat(data)
+	format, mime = textFormat(data)
 	result := classifyText(data)
 	if format != "" {
 		result.Format = format
@@ -126,7 +129,7 @@ func detect(data []byte, prefix bool) Result {
 		result.MIME = mimeText
 	}
 
-	if prefix && prefixResultCanChange(result, len(data)) {
+	if prefix && (binaryNeedsMore || prefixResultCanChange(result, len(data))) {
 		result.Reason = ReasonNeedMore
 	}
 
@@ -135,7 +138,8 @@ func detect(data []byte, prefix bool) Result {
 
 func prefixResultCanChange(result Result, inputLength int) bool {
 	if result.Kind == KindBinary {
-		// sniffLength is also the furthest offset read by a binary signature.
+		// Fixed-offset binary signatures are final once the sniff window is
+		// present. Incomplete PHAR validation is handled before this function.
 		return inputLength < sniffLength
 	}
 	return true

magic_test.go

@@ -135,6 +135,7 @@ func TestDetectAllocations(t *testing.T) {
 		[]byte("package magic\n"),
 		[]byte("\xff\xfeh\x00i\x00"),
 		[]byte("\x89PNG\r\n\x1a\n"),
+		makeNativePHAR(pharTestStub, "", nil, pharTestEntry{name: "file", content: []byte("data")}),
 	}
 	for _, input := range inputs {
 		if allocations := testing.AllocsPerRun(1000, func() {

phar.go

@@ -0,0 +1,152 @@
+package magic
+
+import (
+	"bytes"
+	"encoding/binary"
+)
+
+const (
+	pharHaltCompiler       = "__HALT_COMPILER();"
+	pharManifestFixedLen   = 18
+	pharEntryFixedLen      = 28
+	pharEntryMinLen        = pharEntryFixedLen + 1
+	pharManifestMaxLen     = 100 << 20
+	pharAPIVersionMask     = 0xfff0
+	pharMinimumAPIVersion  = 0x1000
+	pharManifestLengthSize = 4
+	pharClosingTagSize     = 3
+)
+
+type pharStatus uint8
+
+const (
+	pharNotFound pharStatus = iota
+	pharIncomplete
+	pharValid
+)
+
+func nativePHAR(data []byte) pharStatus {
+	stubEnd := bytes.Index(data, []byte(pharHaltCompiler))
+	if stubEnd < 0 {
+		return pharNotFound
+	}
+	stubEnd += len(pharHaltCompiler)
+
+	manifestOffset, status := pharManifestOffset(data, stubEnd)
+	if status != pharValid {
+		return status
+	}
+	if len(data)-manifestOffset < pharManifestLengthSize {
+		return pharIncomplete
+	}
+
+	manifestLength := binary.LittleEndian.Uint32(data[manifestOffset:])
+	if manifestLength < pharManifestFixedLen || manifestLength > pharManifestMaxLen {
+		return pharNotFound
+	}
+
+	manifestStart := manifestOffset + pharManifestLengthSize
+	manifestEnd64 := uint64(manifestStart) + uint64(manifestLength)
+	if manifestEnd64 > uint64(len(data)) {
+		return pharIncomplete
+	}
+	manifestEnd := int(manifestEnd64)
+	manifest := data[manifestStart:manifestEnd]
+
+	entryCount := binary.LittleEndian.Uint32(manifest)
+	if entryCount == 0 {
+		return pharNotFound
+	}
+	apiVersion := binary.BigEndian.Uint16(manifest[4:6])
+	if apiVersion&pharAPIVersionMask < pharMinimumAPIVersion {
+		return pharNotFound
+	}
+
+	offset := 10 // entry count, API version, and global flags
+	aliasLength, ok := pharUint32(manifest, &offset)
+	if !ok || !pharSkip(manifest, &offset, aliasLength) {
+		return pharNotFound
+	}
+	metadataLength, ok := pharUint32(manifest, &offset)
+	if !ok || !pharSkip(manifest, &offset, metadataLength) {
+		return pharNotFound
+	}
+	if uint64(entryCount)*pharEntryMinLen > uint64(len(manifest)-offset) {
+		return pharNotFound
+	}
+
+	var payloadLength uint64
+	for range entryCount {
+		filenameLength, ok := pharUint32(manifest, &offset)
+		if !ok || filenameLength == 0 || !pharSkip(manifest, &offset, filenameLength) {
+			return pharNotFound
+		}
+		if len(manifest)-offset < pharEntryFixedLen-pharManifestLengthSize {
+			return pharNotFound
+		}
+
+		compressedSize := binary.LittleEndian.Uint32(manifest[offset+8:])
+		metadataLength := binary.LittleEndian.Uint32(manifest[offset+20:])
+		offset += pharEntryFixedLen - pharManifestLengthSize
+		if !pharSkip(manifest, &offset, metadataLength) {
+			return pharNotFound
+		}
+		payloadLength += uint64(compressedSize)
+	}
+
+	if uint64(manifestEnd)+payloadLength > uint64(len(data)) {
+		return pharIncomplete
+	}
+	return pharValid
+}
+
+func pharManifestOffset(data []byte, offset int) (int, pharStatus) {
+	if offset >= len(data) {
+		return 0, pharIncomplete
+	}
+	if data[offset] != ' ' && data[offset] != '\n' {
+		return offset, pharValid
+	}
+	if len(data)-offset < pharClosingTagSize {
+		return 0, pharIncomplete
+	}
+	if data[offset+1] != '?' || data[offset+2] != '>' {
+		return offset, pharValid
+	}
+
+	offset += pharClosingTagSize
+	if offset >= len(data) {
+		return 0, pharIncomplete
+	}
+	switch data[offset] {
+	case '\n':
+		offset++
+	case '\r':
+		if offset+1 >= len(data) {
+			return 0, pharIncomplete
+		}
+		if data[offset+1] != '\n' {
+			return 0, pharNotFound
+		}
+		offset += 2
+	}
+	return offset, pharValid
+}
+
+func pharUint32(data []byte, offset *int) (uint32, bool) {
+	if len(data)-*offset < pharManifestLengthSize {
+		return 0, false
+	}
+	value := binary.LittleEndian.Uint32(data[*offset:])
+	*offset += 4
+	return value, true
+}
+
+func pharSkip(data []byte, offset *int, length uint32) bool {
+	end := uint64(*offset) + uint64(length)
+	if end > uint64(len(data)) {
+		return false
+	}
+	*offset = int(end)
+	return true
+}