diff --git a/docs/generator/palo-alto.md b/docs/generator/palo-alto.md index 20219cc0..07d20048 100644 --- a/docs/generator/palo-alto.md +++ b/docs/generator/palo-alto.md @@ -2,12 +2,34 @@ **Class:** Producer (embed-eligible; see [docs/embed.md](../embed.md)) -The Palo Alto generator generates realistic Palo Alto firewall syslog entries in the standard Palo Alto log format. These logs are suitable for testing Palo Alto firewall log processing and analysis systems. +The Palo Alto generator generates realistic PAN-OS firewall syslog entries in the standard comma-separated field format. It emits all 13 log types the Bindplane Palo Alto blueprints parse, each at its full PAN-OS 11.0 field width. -## Example Logs +## Log Types + +Every emitted record is one of these types, selected at random per record. The field count is the number of comma-separated fields in the record body (field 1 is `FUTURE_USE`), validated against Palo Alto's public PAN-OS 11.0 "Syslog Field Descriptions". + +| Log Type | Fields | +|----------|-------:| +| TRAFFIC | 115 | +| THREAT | 121 | +| SYSTEM | 26 | +| CONFIG | 28 | +| AUTHENTICATION | 47 | +| CORRELATION | 22 | +| DECRYPTION | 106 | +| GLOBALPROTECT | 50 | +| GTP | 94 | +| HIP-MATCH | 32 | +| IPTAG | 27 | +| SCTP | 65 | +| USERID | 37 | + +Field order is validated against Palo Alto's public PAN-OS 11.0 "Syslog Field Descriptions" and enforced by tests (`generator/paloalto/logs.go` holds the ordered field-name list per type; `logs_test.go` asserts count and named-position order). The authoritative page is unified across 11.0 and later; for TRAFFIC the page explicitly marks Flow Type, AI Traffic, AI Forward Error, K8S Cluster ID (11.1+) and Adv DevID (12.1.2+), which are excluded. Counts and full field order were extracted from the rendered spec pages (headless Chrome) and are enforced by tests. DECRYPTION 106 and THREAT 121 exclude fields the pages mark 11.1+/12.1.2+ (Cluster Name, Flow Type, AI/K8S/Adv DevID); GTP 94 has none. + +## Example Log (SYSTEM) ``` -1,2024/01/15 10:30:45,001234567890,SYSTEM,threat,2049,2024/01/15 10:30:45,192.0.2.10,10.0.0.1,0.0.0.0,0.0.0.0,rule1,,,inbound,vsys1,trust,untrust,ethernet1/1,ethernet1/2,Forward,1,2024/01/15 10:30:45,123456,1,62873,0,0,0x0,tcp,allow,2049,2024/01/15 10:30:45,10,any,0,1234567890,0x8000000000000000,192.168.1.0-192.168.1.255,United States,0,1,0,aged-out,0,0,0,0,,N/A,0,0,0,0,,,from-policy,,,0,,0,,N/A,unknown,AppThreat-0000,0x0,0x0000000000000000 +Jan 15 10:30:45 1,2024/01/15 10:30:45,001234567890,SYSTEM,general,,2024/01/15 10:30:43,vsys1,42,general,,,general,informational,Config installed,1234567,0x0,0,0,0,0,vsys1,PA-VM,,,2024-01-15T10:30:45.000-05:00 ``` ## Configuration diff --git a/generator/paloalto/logs.go b/generator/paloalto/logs.go new file mode 100644 index 00000000..6e758f91 --- /dev/null +++ b/generator/paloalto/logs.go @@ -0,0 +1,563 @@ +package paloalto + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// PAN-OS 11.0 CSV field ORDER per log type, validated against Palo Alto's public +// "Syslog Field Descriptions" (PAN-OS 11.0). Each list is the exact ordered set +// of field names in the record body; field 1 is FUTURE_USE. buildLogFields +// produces a value slice of the same length, so len(values) == len(names) is the +// field count and the names double as the order contract the tests assert. +// +// The authoritative page is unified across 11.0-and-later and appends/interleaves +// later-version fields. For the four high-width types we exclude ONLY the fields +// the page explicitly marks as introduced after 11.0: +// - 11.1+: Flow Type, Cluster Name, AI Traffic, AI Forward Error, K8S Cluster ID +// - 12.1.2+: Source Adv DevID, Destination Adv DevID +// +// Everything at or before 11.0 is retained. Excluded fields per type are noted +// on each list below. +var logFieldNames = map[string][]string{ + // TRAFFIC 11.0 = 115. From the rendered spec format string (headless Chrome); + // full page = 130, excluded post-11.0 (marked on the page): Flow Type, + // Cluster Name, AI Traffic, AI Forward Error, K8S Cluster ID (11.1+), the TCP + // telemetry block, and Source/Destination Adv DevID (12.1.2+). Kept through + // Offloaded. + "TRAFFIC": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Threat/Content Type", "FUTURE_USE", + "Generated Time", "Source Address", "Destination Address", "NAT Source IP", "NAT Destination IP", + "Rule Name", "Source User", "Destination User", "Application", "Virtual System", "Source Zone", + "Destination Zone", "Inbound Interface", "Outbound Interface", "Log Action", "FUTURE_USE", + "Session ID", "Repeat Count", "Source Port", "Destination Port", "NAT Source Port", + "NAT Destination Port", "Flags", "Protocol", "Action", "Bytes", "Bytes Sent", "Bytes Received", + "Packets", "Start Time", "Elapsed Time", "Category", "FUTURE_USE", "Sequence Number", + "Action Flags", "Source Country", "Destination Country", "FUTURE_USE", "Packets Sent", + "Packets Received", "Session End Reason", "Device Group Hierarchy Level 1", + "Device Group Hierarchy Level 2", "Device Group Hierarchy Level 3", + "Device Group Hierarchy Level 4", "Virtual System Name", "Device Name", "Action Source", + "Source VM UUID", "Destination VM UUID", "Tunnel ID/IMSI", "Monitor Tag/IMEI", + "Parent Session ID", "Parent Start Time", "Tunnel Type", "SCTP Association ID", "SCTP Chunks", + "SCTP Chunks Sent", "SCTP Chunks Received", "Rule UUID", "HTTP/2 Connection", "App Flap Count", + "Policy ID", "Link Switches", "SD-WAN Cluster", "SD-WAN Device Type", "SD-WAN Cluster Type", + "SD-WAN Site", "Dynamic User Group Name", "XFF Address", "Source Device Category", + "Source Device Profile", "Source Device Model", "Source Device Vendor", + "Source Device OS Family", "Source Device OS Version", "Source Hostname", "Source Mac Address", + "Destination Device Category", "Destination Device Profile", "Destination Device Model", + "Destination Device Vendor", "Destination Device OS Family", "Destination Device OS Version", + "Destination Hostname", "Destination Mac Address", "Container ID", "POD Namespace", "POD Name", + "Source External Dynamic List", "Destination External Dynamic List", "Host ID", "Serial Number", + "Source Dynamic Address Group", "Destination Dynamic Address Group", "Session Owner", + "High Resolution Timestamp", "A Slice Service Type", "A Slice Differentiator", + "Application Subcategory", "Application Category", "Application Technology", "Application Risk", + "Application Characteristic", "Application Container", "Tunneled Application", + "Application SaaS", "Application Sanctioned State", "Offloaded", + }, + // THREAT 11.0 = 121. From the rendered spec format string (headless Chrome); + // full page = 123, excluded post-11.0 (marked on the page): Flow Type and + // Cluster Name (both 11.1+). Kept through Cloud Report ID. + "THREAT": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Threat/Content Type", "FUTURE_USE", + "Generated Time", "Source Address", "Destination Address", "NAT Source IP", "NAT Destination IP", + "Rule Name", "Source User", "Destination User", "Application", "Virtual System", "Source Zone", + "Destination Zone", "Inbound Interface", "Outbound Interface", "Log Action", "FUTURE_USE", + "Session ID", "Repeat Count", "Source Port", "Destination Port", "NAT Source Port", + "NAT Destination Port", "Flags", "IP Protocol", "Action", "URL/Filename", "Threat ID", + "Category", "Severity", "Direction", "Sequence Number", "Action Flags", "Source Location", + "Destination Location", "FUTURE_USE", "Content Type", "PCAP_ID", "File Digest", "Cloud", + "URL Index", "User Agent", "File Type", "X-Forwarded-For", "Referer", "Sender", "Subject", + "Recipient", "Report ID", "Device Group Hierarchy Level 1", "Device Group Hierarchy Level 2", + "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4", "Virtual System Name", + "Device Name", "FUTURE_USE", "Source VM UUID", "Destination VM UUID", "HTTP Method", + "Tunnel ID/IMSI", "Monitor Tag/IMEI", "Parent Session ID", "Parent Start Time", "Tunnel Type", + "Threat Category", "Content Version", "FUTURE_USE", "SCTP Association ID", "Payload Protocol ID", + "HTTP Headers", "URL Category List", "Rule UUID", "HTTP/2 Connection", "Dynamic User Group Name", + "XFF Address", "Source Device Category", "Source Device Profile", "Source Device Model", + "Source Device Vendor", "Source Device OS Family", "Source Device OS Version", "Source Hostname", + "Source MAC Address", "Destination Device Category", "Destination Device Profile", + "Destination Device Model", "Destination Device Vendor", "Destination Device OS Family", + "Destination Device OS Version", "Destination Hostname", "Destination MAC Address", + "Container ID", "POD Namespace", "POD Name", "Source External Dynamic List", + "Destination External Dynamic List", "Host ID", "Serial Number", "Domain EDL", + "Source Dynamic Address Group", "Destination Dynamic Address Group", "Partial Hash", + "High Resolution Timestamp", "Reason", "Justification", "A Slice Service Type", + "Application Subcategory", "Application Category", "Application Technology", "Application Risk", + "Application Characteristic", "Application Container", "Tunneled Application", + "Application SaaS", "Application Sanctioned State", "Cloud Report ID", + }, + // SYSTEM 11.0 = 26 (matches published count exactly). + "SYSTEM": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Content/Threat Type", "FUTURE_USE", + "Generated Time", "Virtual System", "Event ID", "Object", "FUTURE_USE", "FUTURE_USE", + "Module", "Severity", "Description", "Sequence Number", "Action Flags", + "Device Group Hierarchy Level 1", "Device Group Hierarchy Level 2", + "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4", "Virtual System Name", + "Device Name", "FUTURE_USE", "FUTURE_USE", "High Resolution Timestamp", + }, + // CONFIG 11.0 = 28 (matches published count exactly). + "CONFIG": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Subtype", "FUTURE_USE", + "Generated Time", "Host", "Virtual System", "Command", "Admin", "Client", "Result", + "Configuration Path", "Before Change Detail", "After Change Detail", "Sequence Number", + "Action Flags", "Device Group Hierarchy Level 1", "Device Group Hierarchy Level 2", + "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4", "Virtual System Name", + "Device Name", "Device Group", "Audit Comment", "FUTURE_USE", "High Resolution Timestamp", + }, + // AUTHENTICATION 11.0 = 47 (matches published count exactly). + "AUTHENTICATION": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Threat/Content Type", "FUTURE_USE", + "Generated Time", "Virtual System", "Source IP", "User", "Normalize User", "Object", + "Authentication Policy", "Repeat Count", "Authentication ID", "Vendor", "Log Action", + "Server Profile", "Description", "Client Type", "Event Type", "Factor Number", + "Sequence Number", "Action Flags", "Device Group Hierarchy Level 1", + "Device Group Hierarchy Level 2", "Device Group Hierarchy Level 3", + "Device Group Hierarchy Level 4", "Virtual System Name", "Device Name", "Virtual System ID", + "Authentication Protocol", "UUID for rule", "High Resolution Timestamp", + "Source Device Category", "Source Device Profile", "Source Device Model", + "Source Device Vendor", "Source Device OS Family", "Source Device OS Version", + "Source Hostname", "Source Mac Address", "Region", "FUTURE_USE", "User Agent", "Session ID", + "Cluster Name", + }, + // CORRELATION 11.0 = 22 (matches published count exactly). + "CORRELATION": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Content/Threat Type", "FUTURE_USE", + "Generated Time", "Source Address", "Source User", "Virtual System", "Category", "Severity", + "Device Group Hierarchy Level 1", "Device Group Hierarchy Level 2", + "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4", "Virtual System Name", + "Device Name", "Virtual System ID", "Object Name", "Object ID", "Evidence", + }, + // DECRYPTION 11.0 = 106. From the rendered spec format string (headless + // Chrome); full page = 107, excluded post-11.0 (marked on the page): Cluster + // Name (11.1+). Kept through Application Sanctioned State. + "DECRYPTION": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Threat/Content Type", "Config Version", + "Generate Time", "Source Address", "Destination Address", "NAT Source IP", "NAT Destination IP", + "Rule", "Source User", "Destination User", "Application", "Virtual System", "Source Zone", + "Destination Zone", "Inbound Interface", "Outbound Interface", "Log Action", "Time Logged", + "Session ID", "Repeat Count", "Source Port", "Destination Port", "NAT Source Port", + "NAT Destination Port", "Flags", "IP Protocol", "Action", "Tunnel", "FUTURE_USE", "FUTURE_USE", + "Source VM UUID", "Destination VM UUID", "UUID for rule", "Stage for Client to Firewall", + "Stage for Firewall to Server", "TLS Version", "Key Exchange Algorithm", "Encryption Algorithm", + "Hash Algorithm", "Policy Name", "Elliptic Curve", "Error Index", "Root Status", "Chain Status", + "Proxy Type", "Certificate Serial Number", "Fingerprint", "Certificate Start Date", + "Certificate End Date", "Certificate Version", "Certificate Size", "Common Name Length", + "Issuer Common Name Length", "Root Common Name Length", "SNI Length", "Certificate Flags", + "Subject Common Name", "Issuer Subject Common Name", "Root Subject Common Name", + "Server Name Indication", "Error", "Container ID", "POD Namespace", "POD Name", + "Source External Dynamic List", "Destination External Dynamic List", + "Source Dynamic Address Group", "Destination Dynamic Address Group", "High Res Timestamp", + "Source Device Category", "Source Device Profile", "Source Device Model", "Source Device Vendor", + "Source Device OS Family", "Source Device OS Version", "Source Hostname", "Source Mac Address", + "Destination Device Category", "Destination Device Profile", "Destination Device Model", + "Destination Device Vendor", "Destination Device OS Family", "Destination Device OS Version", + "Destination Hostname", "Destination Mac Address", "Sequence Number", "Action Flags", + "Device Group Hierarchy Level 1", "Device Group Hierarchy Level 2", + "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4", "Virtual System Name", + "Device Name", "Virtual System ID", "Application Subcategory", "Application Category", + "Application Technology", "Application Risk", "Application Characteristic", + "Application Container", "Application SaaS", "Application Sanctioned State", + }, + // GLOBALPROTECT 11.0 = 50 (matches published count exactly). + "GLOBALPROTECT": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Threat/Content Type", "FUTURE_USE", + "Generated Time", "Virtual System", "Event ID", "Stage", "Authentication Method", "Tunnel Type", + "Source User", "Source Region", "Machine Name", "Public IP", "Public IPv6", "Private IP", + "Private IPv6", "Host ID", "Serial Number", "Client Version", "Client OS", "Client OS Version", + "Repeat Count", "Reason", "Error", "Description", "Status", "Location", "Login Duration", + "Connect Method", "Error Code", "Portal", "Sequence Number", "Action Flags", + "High Res Timestamp", "Selection Type", "Response Time", "Priority", "Attempted Gateways", + "Gateway", "Device Group Hierarchy Level 1", "Device Group Hierarchy Level 2", + "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4", "Virtual System Name", + "Device Name", "Virtual System ID", "Cluster Name", + }, + // GTP 11.0 = 94. From the rendered spec format string (headless Chrome); no + // fields on the page are marked post-11.0. + "GTP": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Threat/Content Type", "FUTURE_USE", + "Generated Time", "Source Address", "Destination Address", "FUTURE_USE", "FUTURE_USE", + "Rule Name", "FUTURE_USE", "FUTURE_USE", "Application", "Virtual System", "Source Zone", + "Destination Zone", "Inbound Interface", "Outbound Interface", "Log Action", "FUTURE_USE", + "Session ID", "FUTURE_USE", "Source Port", "Destination Port", "FUTURE_USE", "FUTURE_USE", + "FUTURE_USE", "Protocol", "Action", "GTP Event Type", "MSISDN", "Access Point Name", + "Radio Access Technology", "GTP Message Type", "End User IP Address", + "Tunnel Endpoint Identifier1", "Tunnel Endpoint Identifier2", "GTP Interface", "GTP Cause", + "Severity", "Serving Country MCC", "Serving Network MNC", "Area Code", "Cell ID", + "GTP Event Code", "FUTURE_USE", "FUTURE_USE", "Source Location", "Destination Location", + "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", + "Tunnel ID/IMSI", "Monitor Tag/IMEI", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", + "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", + "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "Start Time", + "Elapsed Time", "Tunnel Inspection Rule", "Remote User IP", "Remote User ID", "UUID for rule", + "PCAP ID", "High Resolution Timestamp", "A Slice Service Type", "A Slice Differentiator", + "Application Subcategory", "Application Category", "Application Technology", "Application Risk", + "Application Characteristic", "Application Container", "Application SaaS", + "Application Sanctioned State", + }, + // HIP-MATCH 11.0 = 32 (matches published count exactly). + "HIP-MATCH": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Threat/Content Type", "FUTURE_USE", + "Generated Time", "Source User", "Virtual System", "Machine Name", "Operating System", + "Source Address", "HIP", "Repeat Count", "HIP Type", "FUTURE_USE", "FUTURE_USE", + "Sequence Number", "Action Flags", "Device Group Hierarchy Level 1", + "Device Group Hierarchy Level 2", "Device Group Hierarchy Level 3", + "Device Group Hierarchy Level 4", "Virtual System Name", "Device Name", "Virtual System ID", + "IPv6 Source Address", "Host ID", "User Device Serial Number", "Device MAC Address", + "High Resolution Timestamp", "Cluster Name", + }, + // IPTAG 11.0 = 27 (matches published count exactly). + "IPTAG": { + "FUTURE_USE", "Receive Time", "Serial", "Type", "Threat/Content Type", "FUTURE_USE", + "Generate Time", "Virtual System", "Source IP", "Tag Name", "Event ID", "Repeat Count", + "Timeout", "Data Source Name", "Data Source Type", "Data Source Subtype", "Sequence Number", + "Action Flags", "DG Hierarchy Level 1", "DG Hierarchy Level 2", "DG Hierarchy Level 3", + "DG Hierarchy Level 4", "Virtual System Name", "Device Name", "Virtual System ID", + "High Resolution Timestamp", "Cluster Name", + }, + // SCTP 11.0 = 65 (matches published count exactly). + "SCTP": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "FUTURE_USE", "FUTURE_USE", + "Generated Time", "Source Address", "Destination Address", "FUTURE_USE", "FUTURE_USE", + "Rule Name", "FUTURE_USE", "FUTURE_USE", "FUTURE_USE", "Virtual System", "Source Zone", + "Destination Zone", "Inbound Interface", "Outbound Interface", "Log Action", "FUTURE_USE", + "Session ID", "Repeat Count", "Source Port", "Destination Port", "FUTURE_USE", "FUTURE_USE", + "FUTURE_USE", "FUTURE_USE", "IP Protocol", "Action", "Device Group Hierarchy Level 1", + "Device Group Hierarchy Level 2", "Device Group Hierarchy Level 3", + "Device Group Hierarchy Level 4", "Virtual System Name", "Device Name", "Sequence Number", + "FUTURE_USE", "SCTP Association ID", "Payload Protocol ID", "Severity", "SCTP Chunk Type", + "FUTURE_USE", "SCTP Verification Tag 1", "SCTP Verification Tag 2", "SCTP Cause Code", + "Diameter App ID", "Diameter Command Code", "Diameter AVP Code", "SCTP Stream ID", + "SCTP Association End Reason", "Op Code", "SCCP Calling Party SSN", + "SCCP Calling Party Global Title", "SCTP Filter", "SCTP Chunks", "SCTP Chunks Sent", + "SCTP Chunks Received", "Packets", "Packets Sent", "Packets Received", "UUID for rule", + "High Resolution Timestamp", + }, + // USERID 11.0 = 37 (matches published count exactly). + "USERID": { + "FUTURE_USE", "Receive Time", "Serial Number", "Type", "Threat/Content Type", "FUTURE_USE", + "Generated Time", "Virtual System", "Source IP", "User", "Data Source Name", "Event ID", + "Repeat Count", "Time Out Threshold", "Source Port", "Destination Port", "Data Source", + "Data Source Type", "Sequence Number", "Action Flags", "Device Group Hierarchy Level 1", + "Device Group Hierarchy Level 2", "Device Group Hierarchy Level 3", + "Device Group Hierarchy Level 4", "Virtual System Name", "Device Name", "Virtual System ID", + "Factor Type", "Factor Completion Time", "Factor Number", "User Group Flags", "User by Source", + "Tag Name", "High Resolution Timestamp", "Origin Data Source", "FUTURE_USE", "Cluster Name", + }, +} + +// logTypeCatalog is the full set of PAN-OS log types the generator emits, in a +// stable order. +var logTypeCatalog = []string{ + "TRAFFIC", "THREAT", "SYSTEM", "CONFIG", "AUTHENTICATION", "CORRELATION", + "DECRYPTION", "GLOBALPROTECT", "GTP", "HIP-MATCH", "IPTAG", "SCTP", "USERID", +} + +// recordCtx holds per-record values so fields that reference the same entity +// (e.g. the two timestamps, the 5-tuple) stay internally consistent. +type recordCtx struct { + logType string + subtype string + now time.Time + src string + dst string + srcPort string + dstPort string + session string + serial string + proto string + action string + user string + app string +} + +func newCtx(logType string) *recordCtx { + return &recordCtx{ + logType: logType, + subtype: subtypeFor(logType), + now: time.Now(), + src: generateRandomIP(), + dst: generateRandomIP(), + srcPort: generateRandomPort(), + dstPort: generateRandomPort(), + session: generateNumericSessionID(), + serial: generateSerial(), + proto: pick("tcp", "udp"), + action: pick("allow", "deny", "drop", "alert", "reset-both"), + user: pick("corp\\jdoe", "corp\\asmith", ""), + app: pick("web-browsing", "ssl", "dns", "ssh", "smtp"), + } +} + +// buildLogFields returns the ordered CSV field slice for a log type, one value +// per spec field in logFieldNames order. Field 0 is FUTURE_USE ("1"). Values +// are realistic where PAN-OS populates the field, and empty for FUTURE_USE and +// for feature-gated fields that a firewall genuinely emits empty when the +// feature is inactive (Device-ID, EDL/DAG, SD-WAN, SCTP-on-non-SCTP, etc.). +func buildLogFields(logType string) []string { + names, ok := logFieldNames[logType] + if !ok { + names = logFieldNames["SYSTEM"] + logType = "SYSTEM" + } + c := newCtx(logType) + out := make([]string, len(names)) + dgh := 0 // rotate device-group-hierarchy fill so each of the 4 gets "0" + for i, name := range names { + out[i] = c.valueFor(i, name, &dgh) + } + return out +} + +// valueFor maps a spec field name to a realistic value. The first field +// (position 0) is always FUTURE_USE="1", matching real PAN-OS output. +func (c *recordCtx) valueFor(pos int, name string, dgh *int) string { + if pos == 0 { + return "1" + } + switch name { + case "FUTURE_USE": + return "" + case "Receive Time", "Generated Time", "Generate Time", "Time Logged": + return c.now.Format("2006/01/02 15:04:05") + case "Start Time", "Parent Start Time", "Parent Session Start Time", "Factor Completion Time": + return c.now.Add(-time.Duration(randInt(1, 60)) * time.Second).Format("2006/01/02 15:04:05") + case "High Resolution Timestamp", "High Res Timestamp": + return c.now.Format("2006-01-02T15:04:05.000-07:00") + case "Serial Number", "Serial": + return c.serial + case "Type": + return c.logType + case "Threat/Content Type", "Content/Threat Type", "Subtype": + return c.subtype + case "Source Address", "Source IP": + return c.src + case "Destination Address": + return c.dst + case "IPv6 Source Address", "Public IPv6", "Private IPv6", "End User IP Address": + return "" + case "Rule Name", "Rule", "Tunnel Inspection Rule": + return pick("allow-web", "trust-untrust", "block-bad", "default") + case "UUID for rule", "Rule UUID": + return generateRuleUUID() + case "Source User", "User", "Remote User ID", "User by Source", "Normalize User": + return c.user + case "Destination User": + return "" + case "Application": + return c.app + case "Virtual System", "Virtual System Name": + return "vsys1" + case "Virtual System ID": + return "1" + case "Source Zone": + return "trust" + case "Destination Zone": + return "untrust" + case "Inbound Interface": + return "ethernet1/1" + case "Outbound Interface": + return "ethernet1/2" + case "Log Action": + return "log-forwarding-default" + case "Session ID": + return c.session + case "Repeat Count": + return "1" + case "Source Port": + return c.srcPort + case "Destination Port": + return c.dstPort + case "NAT Source Port", "NAT Destination Port": + return "0" + case "NAT Source IP", "NAT Destination IP": + return "0.0.0.0" + case "Flags": + return "0x0" + case "IP Protocol", "Protocol": + return c.proto + case "Action": + return c.action + case "Bytes": + return strconv.Itoa(randInt(200, 200000)) + case "Bytes Sent": + return strconv.Itoa(randInt(100, 100000)) + case "Bytes Received": + return strconv.Itoa(randInt(100, 100000)) + case "Packets": + return strconv.Itoa(randInt(2, 800)) + case "Packets Sent", "Packets Received": + return strconv.Itoa(randInt(1, 400)) + case "Elapsed Time", "Login Duration", "Response Time", "Time Out Threshold", "Timeout": + return strconv.Itoa(randInt(0, 3600)) + case "Category", "Threat Category": + return pick("any", "computer-and-internet-info", "malware", "phishing") + case "Sequence Number": + return strconv.Itoa(randInt(100000, 99999999)) + case "Action Flags": + return "0x0" + case "Source Location", "Source Country", "Source Region", "Serving Country MCC": + return pick("US", "GB", "DE", "10.0.0.0-10.255.255.255") + case "Destination Location", "Destination Country": + return pick("US", "CN", "RU") + case "Session End Reason", "SCTP Association End Reason": + return pick("tcp-rst-from-client", "tcp-fin", "aged-out", "policy-deny") + case "Device Name": + return "PA-VM" + case "Device Group Hierarchy Level 1", "Device Group Hierarchy Level 2", + "Device Group Hierarchy Level 3", "Device Group Hierarchy Level 4", + "DG Hierarchy Level 1", "DG Hierarchy Level 2", "DG Hierarchy Level 3", "DG Hierarchy Level 4": + *dgh++ + return "0" + case "Severity": + return pick("critical", "high", "medium", "low", "informational") + case "Direction": + return pick("client-to-server", "server-to-client") + case "Action Source": + return pick("from-policy", "from-application") + case "URL/Filename": + return pick("http://example.com/x", "evil.example.net/mal.exe", "phish.test/login") + case "Threat ID": + return strconv.Itoa(randInt(10000, 99999)) + case "Content Type": + return pick("", "text/html", "application/pdf") + case "Content Version": + return "AppThreat-" + strconv.Itoa(randInt(8000, 8999)) + "-" + strconv.Itoa(randInt(1000, 9999)) + case "TLS Version": + return pick("TLS1.2", "TLS1.3") + case "Key Exchange Algorithm": + return pick("ECDHE", "RSA") + case "Encryption Algorithm": + return pick("AES-256-GCM", "AES-128-GCM") + case "Hash Algorithm": + return pick("SHA384", "SHA256") + case "Event ID", "GTP Event Code": + return strconv.Itoa(randInt(1, 9999)) + case "Object", "Object Name": + return pick("", "general", "rule1") + case "Object ID": + return strconv.Itoa(randInt(1000, 9999)) + case "Module": + return c.subtype + case "Description", "Evidence", "Reason": + return pick("event recorded", "policy matched", "threshold exceeded") + case "Command": + return pick("commit", "edit", "set", "delete") + case "Admin": + return pick("admin", "operator", "automation") + case "Client", "Client Type": + return pick("Web", "CLI", "API") + case "Result", "Status": + return pick("Submitted", "Succeeded", "success", "failure") + case "Host": + return c.src + case "Machine Name": + return "LAPTOP-" + strconv.Itoa(randInt(1000, 9999)) + case "Operating System", "Client OS": + return pick("Windows 10", "Windows 11", "macOS 14") + case "MSISDN": + return strconv.FormatInt(randInt64(100000000000000, 999999999999999), 10) + case "Access Point Name": + return pick("internet.apn", "ims.apn") + case "Radio Access Technology": + return pick("EUTRAN", "UTRAN", "GERAN") + case "GTP Event Type", "GTP Message Type": + return pick("create-session", "delete-session", "update-bearer") + case "Authentication Method", "Authentication Protocol": + return pick("SAML", "LDAP", "RADIUS", "certificate") + case "Vendor": + return pick("PANW", "Okta", "Duo") + case "Event Type": + return pick("login", "logout") + case "Tunnel Type": + return pick("ipsec", "ssl", "GRE") + case "SCTP Chunk Type": + return pick("DATA", "INIT", "SACK", "HEARTBEAT") + case "HIP": + return pick("firewall-enabled", "disk-encrypted", "av-installed") + case "HIP Type": + return pick("object", "profile") + case "Tag Name": + return pick("malicious", "quarantine", "trusted") + case "Data Source Name", "Data Source", "Origin Data Source": + return pick("AD-agent", "syslog", "XML-API") + case "Data Source Type": + return pick("active-directory", "syslog") + case "Data Source Subtype": + return pick("unknown", "xml-api") + default: + // Feature-gated / optional fields a firewall emits empty when the + // feature is inactive (Device-ID, EDL/DAG, SD-WAN, cert details, + // SCTP-on-non-SCTP, container, etc.). + return "" + } +} + +// subtypeFor returns a representative subtype (field 5) for a log type. +func subtypeFor(logType string) string { + switch logType { + case "TRAFFIC": + return pick("start", "end", "drop", "deny") + case "THREAT": + return pick("url", "vulnerability", "spyware", "virus", "wildfire", "file") + case "SYSTEM": + return pick("general", "auth", "ha", "vpn", "routing") + case "CONFIG": + return "0" + case "AUTHENTICATION": + return "auth" + case "CORRELATION": + return pick("compromised-host", "recon") + case "DECRYPTION": + return pick("ssl", "ssh") + case "GLOBALPROTECT": + return pick("gateway", "portal") + case "GTP": + return pick("gtp-c", "gtp-u", "gtp-prime") + case "HIP-MATCH": + return "hip" + case "IPTAG", "USERID": + return "unknown" + default: + return "" + } +} + +// generatePaloAltoLog builds one PAN-OS syslog line: the BSD syslog timestamp +// header followed by the comma-separated CSV record for a randomly chosen log +// type. Every type in logTypeCatalog is emitted at its full PAN-OS 11.0 field +// width. +func generatePaloAltoLog() string { + logType := logTypeCatalog[randInt(0, len(logTypeCatalog)-1)] + return formatLogLine(logType) +} + +// formatLogLine renders the syslog header plus the comma-joined CSV fields. +func formatLogLine(logType string) string { + timestamp := time.Now().Format("Jan 02 15:04:05") + return timestamp + " " + strings.Join(buildLogFields(logType), ",") +} + +// ---- shared value helpers ---- + +// pick returns one of the provided options at random. +func pick(opts ...string) string { + return opts[randInt(0, len(opts)-1)] +} + +// generateSerial returns a PAN-OS-style 12-digit device serial number. +func generateSerial() string { + var b strings.Builder + for i := 0; i < 12; i++ { + b.WriteString(strconv.Itoa(randInt(0, 9))) + } + return b.String() +} + +// generateRuleUUID returns a random UUIDv4-shaped rule identifier. +func generateRuleUUID() string { + return fmt.Sprintf("%08x-%04x-4%03x-%04x-%012x", + randInt64(0, 0xffffffff), randInt(0, 0xffff), randInt(0, 0xfff), + randInt(0x8000, 0xbfff), randInt64(0, 0xffffffffffff)) +} diff --git a/generator/paloalto/logs_test.go b/generator/paloalto/logs_test.go new file mode 100644 index 00000000..2691feef --- /dev/null +++ b/generator/paloalto/logs_test.go @@ -0,0 +1,153 @@ +package paloalto + +import ( + "net" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// wantFieldCount is the authoritative PAN-OS 11.0 field count per log type, +// validated against Palo Alto's public "Syslog Field Descriptions" (11.0). For +// the four high-width types the count excludes fields the page marks 11.1+/ +// 12.1.2+ (see logFieldNames doc comment). +var wantFieldCount = map[string]int{ + "TRAFFIC": 115, + "THREAT": 121, + "SYSTEM": 26, + "CONFIG": 28, + "AUTHENTICATION": 47, + "CORRELATION": 22, + "DECRYPTION": 106, + "GLOBALPROTECT": 50, + "GTP": 94, + "HIP-MATCH": 32, + "IPTAG": 27, + "SCTP": 65, + "USERID": 37, +} + +func indexOf(names []string, target string) int { + for i, n := range names { + if n == target { + return i + } + } + return -1 +} + +func TestAllThirteenLogTypesPresent(t *testing.T) { + require.Len(t, logTypeCatalog, 13) + for lt := range wantFieldCount { + assert.Contains(t, logTypeCatalog, lt, "catalog missing %s", lt) + assert.Contains(t, logFieldNames, lt, "field-name list missing %s", lt) + } +} + +func TestFieldNameListLengthsMatchSpec(t *testing.T) { + for _, lt := range logTypeCatalog { + assert.Equal(t, wantFieldCount[lt], len(logFieldNames[lt]), + "%s spec field-name list must have %d entries", lt, wantFieldCount[lt]) + } +} + +func TestBuildMatchesNameListLength(t *testing.T) { + for _, lt := range logTypeCatalog { + lt := lt + t.Run(lt, func(t *testing.T) { + for i := 0; i < 20; i++ { + f := buildLogFields(lt) + require.Equal(t, len(logFieldNames[lt]), len(f), + "%s value count must equal its spec field-name count", lt) + } + }) + } +} + +// TestExactFieldOrder pins the spec order at named anchor positions across the +// full width of every type. A field inserted, dropped, or reordered shifts an +// anchor and fails here. +func TestExactFieldOrder(t *testing.T) { + // The shared 7-field header (position, name) is identical for all types + // except field 5, whose meaning is type-specific (kept out of this check). + for _, lt := range logTypeCatalog { + names := logFieldNames[lt] + assert.Equal(t, "FUTURE_USE", names[0], "%s field 1", lt) + assert.Equal(t, "Receive Time", names[1], "%s field 2", lt) + assert.Equal(t, "Type", names[3], "%s field 4 must be Type", lt) + // Field 6 is FUTURE_USE for every type except DECRYPTION (Config Version). + if lt != "DECRYPTION" { + assert.Equal(t, "FUTURE_USE", names[5], "%s field 6", lt) + } else { + assert.Equal(t, "Config Version", names[5], "DECRYPTION field 6") + } + } + + // Per-type anchors deep in the record — these lock tail ordering. + anchors := map[string]map[string]int{ + "TRAFFIC": {"Rule Name": 11, "Protocol": 29, "Source Country": 41, "Session End Reason": 46, "Action Source": 53, "Offloaded": 114}, + "THREAT": {"Rule Name": 11, "Severity": 34, "Report ID": 53, "Cloud Report ID": 120}, + "DECRYPTION": {"Config Version": 5, "TLS Version": 39, "Server Name Indication": 63, "Application Sanctioned State": 105}, + "GTP": {"MSISDN": 32, "Access Point Name": 33, "Application Sanctioned State": 93}, + "SCTP": {"SCTP Association ID": 40, "UUID for rule": 63, "High Resolution Timestamp": 64}, + "USERID": {"Origin Data Source": 34, "Cluster Name": 36}, + } + for lt, want := range anchors { + names := logFieldNames[lt] + for field, pos := range want { + assert.Equal(t, pos, indexOf(names, field), + "%s: %q must be at index %d (position %d)", lt, field, pos, pos+1) + } + } +} + +func TestCommonHeaderValues(t *testing.T) { + for _, lt := range logTypeCatalog { + f := buildLogFields(lt) + assert.Equal(t, "1", f[0], "%s field 1 FUTURE_USE=1", lt) + assert.NotEmpty(t, f[1], "%s field 2 Receive Time", lt) + assert.NotEmpty(t, f[2], "%s field 3 Serial", lt) + assert.Equal(t, lt, f[3], "%s field 4 Type", lt) + assert.NotEmpty(t, f[6], "%s field 7 Generated Time", lt) + } +} + +// TestFutureUseFieldsEmpty asserts every FUTURE_USE position (except field 1, +// which PAN-OS emits as "1") renders empty. +func TestFutureUseFieldsEmpty(t *testing.T) { + for _, lt := range logTypeCatalog { + names := logFieldNames[lt] + f := buildLogFields(lt) + for i, name := range names { + if i == 0 || name != "FUTURE_USE" { + continue + } + assert.Empty(t, f[i], "%s FUTURE_USE at position %d must be empty", lt, i+1) + } + } +} + +// TestKeyFieldsPopulated checks that value-bearing fields carry realistic +// values at their spec position (order-linked: value read via field name index). +func TestKeyFieldsPopulated(t *testing.T) { + f := buildLogFields("TRAFFIC") + names := logFieldNames["TRAFFIC"] + assert.NotEmpty(t, f[indexOf(names, "Rule Name")], "TRAFFIC Rule Name (pos 12)") + assert.NotNil(t, net.ParseIP(f[indexOf(names, "Source Address")]), "TRAFFIC Source Address must be an IP") + assert.NotNil(t, net.ParseIP(f[indexOf(names, "Destination Address")]), "TRAFFIC Destination Address must be an IP") + + f = buildLogFields("USERID") + assert.Equal(t, "1", f[0], "USERID field 1 must be FUTURE_USE=1 (blueprint FUTURE_USER is a bug)") +} + +func TestFormatLogLineFieldCount(t *testing.T) { + for _, lt := range logTypeCatalog { + line := formatLogLine(lt) + idx := strings.Index(line, " 1,") + require.GreaterOrEqual(t, idx, 0, "%s line must contain the CSV start", lt) + fields := strings.Split(line[idx+1:], ",") + assert.Equal(t, wantFieldCount[lt], len(fields), "%s CSV field count", lt) + } +} diff --git a/generator/paloalto/paloalto.go b/generator/paloalto/paloalto.go index 3229f1f1..359372e6 100644 --- a/generator/paloalto/paloalto.go +++ b/generator/paloalto/paloalto.go @@ -202,118 +202,6 @@ func (g *Generator) recordWriteError(errorType string, _ error) { // ---- Palo Alto log synthesis ---- -func generatePaloAltoLog() string { - now := time.Now() - timestamp := now.Format("Jan 02 15:04:05") - dateTime := now.Format("2006/01/02 15:04:05") - // Generate an older timestamp for some fields - oldTime := now.Add(-time.Duration(randInt(1, 365*24)) * time.Hour) - oldDateTime := oldTime.Format("2006/01/02 15:04:05") - - logTypes := []string{"TRAFFIC", "THREAT", "SYSTEM", "CONFIG"} - logType := logTypes[randInt(0, len(logTypes)-1)] - - sessionID := generateNumericSessionID() - - switch logType { - case "SYSTEM": - return generateSystemLog(timestamp, dateTime, oldDateTime, sessionID) - case "CONFIG": - return generateConfigLog(timestamp, dateTime, oldDateTime, sessionID) - case "TRAFFIC": - return generateTrafficLog(timestamp, dateTime, sessionID) - case "THREAT": - return generateThreatLog(timestamp, dateTime, sessionID) - default: - return generateSystemLog(timestamp, dateTime, oldDateTime, sessionID) - } -} - -func generateSystemLog(timestamp, dateTime, oldDateTime, sessionID string) string { - subtypes := []string{"general", "ras", "vpn", "routing"} - subtype := subtypes[randInt(0, len(subtypes)-1)] - - messages := []string{ - "Config installed", - "RASMGR daemon configuration load phase-1 succeeded.", - "RASMGR daemon configuration load phase-2 succeeded.", - "IKE daemon configuration load phase-1 succeeded.", - "IKE daemon configuration load phase-2 succeeded.", - "Route daemon configuration load phase-1 succeeded.", - "Route daemon configuration load phase-2 succeeded.", - "Log type config cleared by user", - } - message := messages[randInt(0, len(messages)-1)] - - configNum := randInt(800, 1000) - return fmt.Sprintf("%s 1,%s,%s,SYSTEM,%s,1,%s,,unknown,,0,0,general,informational,%s,%d,0x0", - timestamp, dateTime, sessionID, subtype, oldDateTime, message, configNum) -} - -func generateConfigLog(timestamp, dateTime, oldDateTime, sessionID string) string { - actions := []string{"commit", "edit", "set"} - action := actions[randInt(0, len(actions)-1)] - - users := []string{"admin", "badguy", "operator"} - user := users[randInt(0, len(users)-1)] - - methods := []string{"Web", "CLI"} - method := methods[randInt(0, len(methods)-1)] - - statuses := []string{"Submitted", "Succeeded"} - status := statuses[randInt(0, len(statuses)-1)] - - details := []string{ - "", - " vsys vsys1 profiles data-objects PII", - " config shared local-user-database user badguy", - " config mgt-config users badguy", - " vsys vsys1 profiles url-filtering monzyspolicy", - } - detail := details[randInt(0, len(details)-1)] - - ip := generateRandomIP() - return fmt.Sprintf("%s 1,%s,%s,CONFIG,0,0,%s,%s,,%s,%s,%s,%s%s,0,0x0", - timestamp, dateTime, sessionID, oldDateTime, ip, action, user, method, status, detail) -} - -func generateTrafficLog(timestamp, dateTime, sessionID string) string { - actions := []string{"allow", "deny", "drop"} - action := actions[randInt(0, len(actions)-1)] - - sourceIP := generateRandomIP() - destIP := generateRandomIP() - sourcePort := generateRandomPort() - destPort := generateRandomPort() - - protocols := []string{"tcp", "udp", "icmp"} - protocol := protocols[randInt(0, len(protocols)-1)] - - bytes := randInt(50, 1000) - packets := randInt(1, 10) - - seqNum := randInt(100000, 999999) - seqNum2 := randInt(100000, 999999) - return fmt.Sprintf("%s 1,%s,%s,TRAFFIC,%s,2049,%s,%s,%s,%s,%s,,,incomplete,vsys1,untrusted,trusted,ethernet1/3,ethernet1/2,log-forwarding-default,%s,%d,1,%s,%s,%s,%s,0x400064,%s,%s,%d,%d,0,4,%d,7,any,0,0x0,0,4,0,aged-out,0,0,0,0,,from-policy,,,0,,0,,N/A,0,0,0,0", - timestamp, dateTime, sessionID, action, dateTime, sourceIP, destIP, sourceIP, destIP, dateTime, seqNum, sourcePort, destPort, sourcePort, destPort, protocol, action, bytes, packets, seqNum2) -} - -func generateThreatLog(timestamp, dateTime, sessionID string) string { - sourceIP := generateRandomIP() - destIP := generateRandomIP() - sourcePort := generateRandomPort() - destPort := generateRandomPort() - - protocols := []string{"tcp", "udp"} - protocol := protocols[randInt(0, len(protocols)-1)] - - domains := []string{"example.com", "malicious-site.com", "suspicious-domain.net"} - domain := domains[randInt(0, len(domains)-1)] - - return fmt.Sprintf("%s 1,%s,%s,THREAT,url,1,%s,%s,%s,%s,%s,RFC1918 to Internet,,,web-browsing,vsys1,Trust,Untrust,ae1.902,ae1.1000,LoggingToPanorama,%s,%d,1,%s,%s,%s,%s,0x408000,%s,alert,\"%s\",(9999),not-defined,informational,client-to-server,%d,0x0,%s,0,text/html", - timestamp, dateTime, sessionID, dateTime, sourceIP, destIP, sourceIP, destIP, dateTime, randInt(100000, 999999), sourcePort, destPort, sourcePort, destPort, protocol, domain, randInt64(1000000000, 9999999999), sourceIP) -} - func generateRandomIP() string { ranges := []string{ "10.0.0.0/8",