diff --git a/docker/README.md b/docker/README.md index 141dabd0..47897610 100644 --- a/docker/README.md +++ b/docker/README.md @@ -109,6 +109,7 @@ docker compose -f docker/docker-compose.telemetry-generator.yml up -d | `blitz-hostmetrics-windows` | Host Metrics | Synthetic host metrics (CPU, memory, disk, etc.) for Windows | | `blitz-traces` | Traces | Synthetic distributed traces (HTTP + DB spans) | | `blitz-fix` | FIX | FIX protocol messages (4.2 / 4.4 / 5.0 SP2) across 10 asset categories — NewOrderSingle, ExecutionReport, cancel/replace/status | +| `blitz-f5` | F5 | Multi-product F5 syslog: BIG-IP LTM/ASM/AFM/APM/DNS/audit, NGINX-on-F5 Plus + App Protect, iRules, F5OS | ## Running Individual Generators diff --git a/docker/docker-compose.telemetry-generator.yml b/docker/docker-compose.telemetry-generator.yml index 7780a7e2..9ecb0b5e 100644 --- a/docker/docker-compose.telemetry-generator.yml +++ b/docker/docker-compose.telemetry-generator.yml @@ -230,6 +230,19 @@ services: BLITZ_OUTPUT_OTLPGRPC_HOST: bdot-collector BLITZ_OUTPUT_OTLPGRPC_PORT: "4317" + # Multi-product F5 log generator (BIG-IP LTM/ASM/AFM/APM/DNS/audit, + # NGINX-on-F5 Plus + App Protect, iRules, F5OS). All products enabled + # by default; set BLITZ_GENERATOR_F5_ENABLEDPRODUCTS to a subset. + blitz-f5: + <<: *blitz-common + environment: + BLITZ_GENERATOR_TYPE: f5 + BLITZ_GENERATOR_F5_WORKERS: ${BLITZ_WORKERS:-1} + BLITZ_GENERATOR_F5_RATE: ${BLITZ_RATE:-1s} + BLITZ_OUTPUT_TYPE: otlp-grpc + BLITZ_OUTPUT_OTLPGRPC_HOST: bdot-collector + BLITZ_OUTPUT_OTLPGRPC_PORT: "4317" + networks: telemetry-net: driver: bridge diff --git a/docs/generator/f5.md b/docs/generator/f5.md new file mode 100644 index 00000000..ffd4becd --- /dev/null +++ b/docs/generator/f5.md @@ -0,0 +1,122 @@ +# F5 Generator + +The F5 generator emits a weighted mix of syslog-shaped log lines across the F5 product portfolio. It is a single generator (`generator/f5`) with one subpackage per product, mirroring the FIX generator: adding a product later is a matter of a new subpackage that self-registers into the catalog. + +All products are enabled by default. Restrict the set with `enabledProducts`, and shift the mix with `weights`. Output is deterministic from `seed` (negative = randomize per worker; 0+ = byte-identical across runs). + +## Products + +| Token | Product | Shape | +|-------|---------|-------| +| `ltm` | BIG-IP LTM (Local Traffic Manager) | TMM request log, combined-style with virtual-server/pool context | +| `asm` | BIG-IP ASM (Application Security Manager) | WAF security event, comma-separated `key="value"` fields | +| `afm` | BIG-IP AFM (Advanced Firewall Manager) | L3/L4 firewall event, `key="value"` fields | +| `apm` | BIG-IP APM (Access Policy Manager) | Access / auth / SSO log with MCP-style code + session | +| `dns` | BIG-IP DNS (formerly GTM) | GSLB / DNS query-resolution log | +| `audit` | BIG-IP audit | `mcpd` configuration audit record (`AUDIT -` format) | +| `nginx-plus` | NGINX-on-F5 (NGINX Plus) | Access log (combined + Plus upstream fields) and error log | +| `nginx-app-protect` | NGINX App Protect (WAF) | Security event, `key="value"` fields | +| `irules` | iRules logging | Operator log line (`Rule /Common/ :`) | +| `f5os` | F5OS / TMOS platform | Platform audit + system events (chassis, tenant, service) | + +## Field fidelity and references + +F5 logging is **operator-configurable**: the field set and order are chosen by the administrator (BIG-IP logging profiles / storage-format Field-List, NGINX `log_format`, iRules `log` statements). So the fidelity target is F5's **documented default format** for each product. Products with a published, enumerable default are **byte-exact to that default** (field set + order asserted by test). Products whose default is a free-text or fully operator-defined template are a **realistic instance of the documented default profile** (config-dependent, cited). + +| Product | Fidelity | Field count | Reference URL | +|---------|----------|-------------|---------------| +| `nginx-app-protect` | byte-exact to default | 41 | https://docs.nginx.com/waf/logging/security-logs/ | +| `asm` | byte-exact to documented default syslog set | 16 | https://techdocs.f5.com/en-us/bigip-17-5-0/big-ip-asm-implementations/logging-application-security-events.html | +| `afm` | byte-exact to default ("None") format | 14 | https://techdocs.f5.com/kb/en-us/products/big-ip-afm/manuals/product/network-firewall-policies-implementations-11-6-0/13.html | +| `nginx-plus` | byte-exact to combined + Plus upstream vars | 8 + 5 | https://docs.nginx.com/nginx/admin-guide/monitoring/logging/ | +| `audit` | byte-exact to documented `mcpd` AUDIT template | template | https://techdocs.f5.com/en-us/bigip-17-5-0/external-monitoring-of-big-ip-systems-implementations.html | +| `ltm` | default-instance (config-dependent) | operator-defined | https://techdocs.f5.com/en-us/bigip-17-5-0/big-ip-ltm-implementations/configuring-request-logging.html | +| `apm` | default-instance (config-dependent) | operator-defined | https://techdocs.f5.com/en-us/bigip-17-5-0/big-ip-access-policy-manager-visual-policy-editor.html | +| `dns` | default-instance (config-dependent) | operator-defined | https://techdocs.f5.com/en-us/bigip-17-5-0/external-monitoring-of-big-ip-systems-implementations.html | +| `irules` | default-instance (config-dependent) | operator-defined | https://clouddocs.f5.com/api/irules/log.html | +| `f5os` | default-instance (config-dependent) | operator-defined | https://techdocs.f5.com/en-us/f5os-a-1-8-0/f5-rseries-systems-administration-configuration.html | + +The five byte-exact products declare their default field order in code (`DefaultFields()` / `AccessFormatVars()`) and assert the emitted field set/order against it in tests. The five default-instance products emit a realistic instance of F5's documented default profile and are marked config-dependent in their package docs, since F5 publishes no fixed positional spec for them. + +## Configuration + +YAML (standalone CLI): + +```yaml +generator: + type: f5 + f5: + workers: 2 + rate: 500ms + hostname: bigip-prod-01 # syslog-header device hostname + enabledProducts: # omit / leave empty for all 10 + - ltm + - asm + - afm + weights: # omit for an equal mix + asm: 5 + ltm: 2 + seed: 42 # >=0 deterministic, <0 randomize +``` + +Environment variables map to these paths with the `BLITZ_` prefix (e.g. `BLITZ_GENERATOR_F5_WORKERS`, `BLITZ_GENERATOR_F5_RATE`), as used by the docker compose service. + +Programmatic (`f5.Config` in Go): + +```go +type Config struct { + Workers int // parallel emission workers + Rate time.Duration // 1 line per Rate per worker + Hostname string // syslog-header device hostname + EnabledProducts []string // empty = all 10 products + Weights map[string]float64 // per-product mix ratio; absent = 1 + Seed int64 // >=0 deterministic, <0 randomize +} +``` + +## Example Configuration + +### All products, equal mix + +```yaml +generator: + type: f5 + f5: + workers: 2 + rate: 500ms + hostname: bigip-prod-01 +``` + +### WAF-focused subset with a weighted mix + +```yaml +generator: + type: f5 + f5: + enabledProducts: [asm, nginx-app-protect, afm] + weights: + asm: 5 + nginx-app-protect: 3 + afm: 1 + seed: 42 +``` + +## Example Output + +LTM request log: + +``` +<134>Sep 24 15:04:05 bigip1 tmm[4211]: 203.0.113.7 - - [24/Sep/2026:15:04:05 +0000] "GET /api/v1/orders HTTP/1.1" 200 8231 "-" "curl/8.4.0" vs=/Common/vs_https_443 pool=/Common/pool_web member=10.2.3.4:8080 +``` + +ASM security event (truncated): + +``` +<134>Sep 24 15:04:05 bigip1 ASM: unit_hostname="bigip1",policy_name="/Common/prod_waf_policy",violations="SQL-Injection",support_id="1234567890123456",request_status="blocked",method="POST",attack_type="SQL-Injection",severity="Critical",... +``` + +Audit record: + +``` +<133>Sep 24 15:04:05 bigip1 mcpd[4102]: 01070417:5: AUDIT - client tmsh, user admin - transaction #4211-1 - object 0 - modify ltm virtual /Common/vs_https_443 { ... } from 10.1.1.1 +``` diff --git a/generator/f5/catalog/product.go b/generator/f5/catalog/product.go new file mode 100644 index 00000000..7579e126 --- /dev/null +++ b/generator/f5/catalog/product.go @@ -0,0 +1,29 @@ +// Package catalog holds the F5 product registry. Each per-product +// subpackage under generator/f5/products registers one Product at init +// time via Register; the top-level f5 generator reads them back to emit +// a weighted mix of F5 log lines. Adding a product is a matter of a new +// subpackage that self-registers — no change to the generator core. +package catalog + +import ( + "math/rand" + "time" +) + +// Ctx carries the per-line context a product's Build func needs: the +// emission time and the emitting device hostname. All randomness comes +// from the supplied *rand.Rand so output is deterministic from seed. +type Ctx struct { + // Now is the timestamp for the emitted record. + Now time.Time + // Hostname is the simulated F5 device hostname. + Hostname string +} + +// Product is one F5 product's log emitter. Name is the config token +// (e.g. "ltm", "asm"). Build returns one syslog-shaped log line for the +// product, drawing all randomness from r. +type Product struct { + Name string + Build func(r *rand.Rand, c *Ctx) string +} diff --git a/generator/f5/catalog/registry.go b/generator/f5/catalog/registry.go new file mode 100644 index 00000000..8f431424 --- /dev/null +++ b/generator/f5/catalog/registry.go @@ -0,0 +1,54 @@ +package catalog + +import ( + "sort" + "sync" +) + +// registry holds all registered Products keyed by Name. Per-product +// subpackages register at init (single-goroutine); the generator reads +// them on construction. Guarded by an RWMutex for safety. +var ( + mu sync.RWMutex + products = map[string]Product{} +) + +// Register adds a Product to the global registry. Panics on a duplicate +// Name — that signals a programmer error in the product catalog, not a +// recoverable runtime condition. +func Register(p Product) { + mu.Lock() + defer mu.Unlock() + if _, exists := products[p.Name]; exists { + panic("f5 catalog: duplicate product registration for " + p.Name) + } + products[p.Name] = p +} + +// Get returns the Product registered under name, or ok=false. +func Get(name string) (Product, bool) { + mu.RLock() + defer mu.RUnlock() + p, ok := products[name] + return p, ok +} + +// AllProducts returns every registered Product, sorted by Name for +// deterministic ordering. +func AllProducts() []Product { + mu.RLock() + defer mu.RUnlock() + out := make([]Product, 0, len(products)) + for _, p := range products { + out = append(out, p) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +// ResetForTest empties the registry. Tests only. +func ResetForTest() { + mu.Lock() + defer mu.Unlock() + products = map[string]Product{} +} diff --git a/generator/f5/catalog/registry_test.go b/generator/f5/catalog/registry_test.go new file mode 100644 index 00000000..9d87e86d --- /dev/null +++ b/generator/f5/catalog/registry_test.go @@ -0,0 +1,44 @@ +package catalog + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRegisterAndGet(t *testing.T) { + ResetForTest() + p := Product{Name: "ltm", Build: func(_ *rand.Rand, _ *Ctx) string { return "line" }} + Register(p) + + got, ok := Get("ltm") + require.True(t, ok) + require.Equal(t, "ltm", got.Name) + require.Equal(t, "line", got.Build(rand.New(rand.NewSource(1)), &Ctx{})) +} + +func TestRegisterDuplicatePanics(t *testing.T) { + ResetForTest() + Register(Product{Name: "ltm", Build: func(_ *rand.Rand, _ *Ctx) string { return "" }}) + require.Panics(t, func() { + Register(Product{Name: "ltm", Build: func(_ *rand.Rand, _ *Ctx) string { return "" }}) + }) +} + +func TestAllProductsSortedByName(t *testing.T) { + ResetForTest() + Register(Product{Name: "zzz", Build: func(_ *rand.Rand, _ *Ctx) string { return "" }}) + Register(Product{Name: "aaa", Build: func(_ *rand.Rand, _ *Ctx) string { return "" }}) + + all := AllProducts() + require.Len(t, all, 2) + require.Equal(t, "aaa", all[0].Name) + require.Equal(t, "zzz", all[1].Name) +} + +func TestGetMissing(t *testing.T) { + ResetForTest() + _, ok := Get("nope") + require.False(t, ok) +} diff --git a/generator/f5/catalog/syslog.go b/generator/f5/catalog/syslog.go new file mode 100644 index 00000000..372a3cc3 --- /dev/null +++ b/generator/f5/catalog/syslog.go @@ -0,0 +1,20 @@ +package catalog + +import ( + "fmt" + "time" +) + +// bsdStamp is the BSD-syslog (RFC 3164) timestamp layout: month, space- +// padded day, time. F5 devices emit this on the syslog wire. +const bsdStamp = "Jan _2 15:04:05" + +// SyslogHeader builds an RFC 3164 header: "timestamp host tag[pid]:". +// A pid <= 0 omits the "[pid]" segment (used by daemons like mcpd that +// log without one in some records). +func SyslogHeader(pri int, now time.Time, host, tag string, pid int) string { + if pid <= 0 { + return fmt.Sprintf("<%d>%s %s %s:", pri, now.Format(bsdStamp), host, tag) + } + return fmt.Sprintf("<%d>%s %s %s[%d]:", pri, now.Format(bsdStamp), host, tag, pid) +} diff --git a/generator/f5/catalog/syslog_test.go b/generator/f5/catalog/syslog_test.go new file mode 100644 index 00000000..4fc7c6ff --- /dev/null +++ b/generator/f5/catalog/syslog_test.go @@ -0,0 +1,20 @@ +package catalog + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSyslogHeader(t *testing.T) { + now := time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC) + got := SyslogHeader(134, now, "bigip1", "tmm", 4211) + require.Equal(t, "<134>Sep 24 15:04:05 bigip1 tmm[4211]:", got) +} + +func TestSyslogHeaderNoPID(t *testing.T) { + now := time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC) + got := SyslogHeader(134, now, "bigip1", "mcpd", 0) + require.Equal(t, "<134>Sep 24 15:04:05 bigip1 mcpd:", got) +} diff --git a/generator/f5/f5.go b/generator/f5/f5.go new file mode 100644 index 00000000..1e03a7bf --- /dev/null +++ b/generator/f5/f5.go @@ -0,0 +1,278 @@ +// Package f5 is the multi-product F5 log generator. It emits a weighted +// mix of syslog-shaped log lines across the F5 product portfolio +// (BIG-IP LTM/ASM/AFM/APM/DNS/audit, NGINX-on-F5 Plus + App Protect, +// iRules, F5OS/TMOS), each product modeled in its own subpackage under +// generator/f5/products and self-registered into generator/f5/catalog. +// +// Architecture mirrors the FIX generator: a catalog of registered +// products, per-product subpackages, and this top-level Generator that +// spawns workers each running a deterministic emit loop seeded from the +// user's Seed and pushing records into an embed.LogConsumer. +// +// Determinism: same Seed + same product set = same output stream per +// worker. +package f5 + +import ( + "context" + "fmt" + "math/rand" + "sort" + "sync" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.uber.org/zap" + + "github.com/observiq/blitz/embed" + "github.com/observiq/blitz/generator" + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/generator/resource" + "github.com/observiq/blitz/internal/datagen" + "github.com/observiq/blitz/telemetry" + + // Register every F5 product. + _ "github.com/observiq/blitz/generator/f5/products/afm" + _ "github.com/observiq/blitz/generator/f5/products/apm" + _ "github.com/observiq/blitz/generator/f5/products/appprotect" + _ "github.com/observiq/blitz/generator/f5/products/asm" + _ "github.com/observiq/blitz/generator/f5/products/audit" + _ "github.com/observiq/blitz/generator/f5/products/dns" + _ "github.com/observiq/blitz/generator/f5/products/f5os" + _ "github.com/observiq/blitz/generator/f5/products/irules" + _ "github.com/observiq/blitz/generator/f5/products/ltm" + _ "github.com/observiq/blitz/generator/f5/products/nginxplus" +) + +const componentName = "f5" + +// defaultHostname is the simulated F5 device hostname when none is configured. +const defaultHostname = "bigip1" + +// Config configures the F5 generator. +type Config struct { + // Workers spawned for parallel emission. Each worker has its own RNG. + Workers int + // Rate is the per-worker emission interval (one line per Rate). + Rate time.Duration + // Hostname is the simulated F5 device hostname in the syslog header. + Hostname string + // EnabledProducts restricts emission to a subset of product names + // (e.g. "ltm", "asm"). Empty = all registered products. + EnabledProducts []string + // Weights sets the relative mix ratio per product name. A product + // absent from the map (or the whole map empty) defaults to weight 1. + Weights map[string]float64 + // Seed is the base RNG seed. Negative = randomize per worker; 0+ = + // deterministic (worker N gets Seed+N). + Seed int64 +} + +// DefaultConfig returns a Config with sensible defaults: one worker, 1s +// rate, all products at equal weight, randomized seed. +func DefaultConfig() Config { + return Config{Workers: 1, Rate: time.Second, Hostname: defaultHostname, Seed: -1} +} + +// Generator emits F5 log lines at the configured rate. +type Generator struct { + embed.ProducerMarker + + logger *zap.Logger + cfg Config + consumer embed.LogConsumer + static *resource.StaticResources + metrics *generator.Metrics + + products []catalog.Product + cumWeit []float64 // cumulative weights, parallel to products + total float64 + + wg sync.WaitGroup + stopCh chan struct{} +} + +// New constructs an F5 Generator. Returns an error for invalid inputs +// (nil logger/consumer, workers < 1, non-positive rate) or an +// EnabledProducts entry that names no registered product. +func New(logger *zap.Logger, cfg Config, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*Generator, error) { + if logger == nil { + return nil, fmt.Errorf("logger cannot be nil") + } + if consumer == nil { + return nil, fmt.Errorf("consumer cannot be nil") + } + if cfg.Workers < 1 { + return nil, fmt.Errorf("workers must be 1 or greater, got %d", cfg.Workers) + } + if cfg.Rate <= 0 { + return nil, fmt.Errorf("rate must be positive, got %v", cfg.Rate) + } + if cfg.Hostname == "" { + cfg.Hostname = defaultHostname + } + + products, err := selectProducts(cfg.EnabledProducts) + if err != nil { + return nil, err + } + + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + + g := &Generator{ + logger: logger, + cfg: cfg, + consumer: consumer, + static: resource.FromIdentity(nil, componentName), + metrics: metrics, + products: products, + stopCh: make(chan struct{}), + } + g.buildWeights() + return g, nil +} + +// selectProducts returns the enabled products (all when enabled is empty), +// erroring on any name that is not registered. +func selectProducts(enabled []string) ([]catalog.Product, error) { + if len(enabled) == 0 { + return catalog.AllProducts(), nil + } + out := make([]catalog.Product, 0, len(enabled)) + for _, name := range enabled { + p, ok := catalog.Get(name) + if !ok { + return nil, fmt.Errorf("unknown f5 product %q", name) + } + out = append(out, p) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, nil +} + +// buildWeights precomputes the cumulative-weight table for weighted product +// selection. A product missing from Weights (or a non-positive weight) +// defaults to 1. +func (g *Generator) buildWeights() { + g.cumWeit = make([]float64, len(g.products)) + var acc float64 + for i, p := range g.products { + w := 1.0 + if wv, ok := g.cfg.Weights[p.Name]; ok && wv > 0 { + w = wv + } + acc += w + g.cumWeit[i] = acc + } + g.total = acc +} + +// pickProduct selects a product by weighted random draw from r. +func (g *Generator) pickProduct(r *rand.Rand) catalog.Product { + x := r.Float64() * g.total + for i, cum := range g.cumWeit { + if x < cum { + return g.products[i] + } + } + return g.products[len(g.products)-1] +} + +// Name returns the module identifier. +func (g *Generator) Name() string { return componentName } + +// SetHostIdentity sets the simulated host whose identity every emitted record +// carries (PIPE-1036). A nil identity keeps the process-hostname fallback. +func (g *Generator) SetHostIdentity(id *datagen.SystemIdentity) { + g.static = resource.FromIdentity(id, componentName) +} + +// Start launches the worker goroutines. +func (g *Generator) Start(_ context.Context) error { + g.logger.Info("Starting F5 generator", + zap.Int("workers", g.cfg.Workers), + zap.Duration("rate", g.cfg.Rate), + zap.Int("products", len(g.products)), + ) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.cfg.Workers), componentName) + for i := 0; i < g.cfg.Workers; i++ { + g.wg.Add(1) + go g.runWorker(i) // #nosec G118 -- workers bounded by Stop() and the WaitGroup + } + return nil +} + +// Stop signals workers to drain and waits for them to exit. +func (g *Generator) Stop(ctx context.Context) error { + g.logger.Info("Stopping F5 generator") + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) + close(g.stopCh) + + done := make(chan struct{}) + go func() { + g.wg.Wait() + close(done) + }() + + select { + case <-done: + return nil + case <-ctx.Done(): + return fmt.Errorf("stop cancelled due to context cancellation: %w", ctx.Err()) + } +} + +func (g *Generator) runWorker(workerIdx int) { + defer g.wg.Done() + + seed := g.cfg.Seed + if seed < 0 { + seed = time.Now().UnixNano() + int64(workerIdx) + } else { + seed += int64(workerIdx) + } + r := rand.New(rand.NewSource(seed)) // #nosec G404 -- seeded for determinism contract + + ticker := time.NewTicker(g.cfg.Rate) + defer ticker.Stop() + + for { + select { + case <-g.stopCh: + return + case <-ticker.C: + line := g.buildLine(r) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + rec := embed.LogRecord{ + Message: line, + Metadata: embed.LogRecordMetadata{ + Severity: "INFO", + Resource: g.static.Record(), + }, + } + if err := g.consumer.ConsumeLogs(ctx, []embed.LogRecord{rec}); err != nil { + g.logger.Debug("F5 emit failed", zap.Error(err)) + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", "consume"))), + ) + } + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + cancel() + } + } +} + +// buildLine picks a weighted product and builds one log line from it. +func (g *Generator) buildLine(r *rand.Rand) string { + p := g.pickProduct(r) + return p.Build(r, &catalog.Ctx{Now: time.Now(), Hostname: g.cfg.Hostname}) +} + +// SupportedTelemetry reports that this generator produces logs. +func (g *Generator) SupportedTelemetry() []telemetry.Type { + return []telemetry.Type{telemetry.Logs} +} diff --git a/generator/f5/f5_test.go b/generator/f5/f5_test.go new file mode 100644 index 00000000..e688ad1e --- /dev/null +++ b/generator/f5/f5_test.go @@ -0,0 +1,110 @@ +package f5 + +import ( + "context" + "math/rand" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/observiq/blitz/embed" + "github.com/observiq/blitz/telemetry" +) + +func rngFor(seed int64) *rand.Rand { return rand.New(rand.NewSource(seed)) } + +type captureConsumer struct { + mu sync.Mutex + got []string +} + +func (c *captureConsumer) ConsumeLogs(_ context.Context, records []embed.LogRecord) error { + c.mu.Lock() + defer c.mu.Unlock() + for _, r := range records { + c.got = append(c.got, r.Message) + } + return nil +} + +func (c *captureConsumer) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.got) +} + +func newGen(t *testing.T, cfg Config) (*Generator, *captureConsumer) { + t.Helper() + cc := &captureConsumer{} + g, err := New(zap.NewNop(), cfg, cc, embed.TelemetrySettings{}) + require.NoError(t, err) + return g, cc +} + +func TestNewValidates(t *testing.T) { + cc := &captureConsumer{} + _, err := New(nil, DefaultConfig(), cc, embed.TelemetrySettings{}) + require.Error(t, err) + _, err = New(zap.NewNop(), DefaultConfig(), nil, embed.TelemetrySettings{}) + require.Error(t, err) + _, err = New(zap.NewNop(), Config{Workers: 0, Rate: time.Second}, cc, embed.TelemetrySettings{}) + require.Error(t, err) + _, err = New(zap.NewNop(), Config{Workers: 1, Rate: 0}, cc, embed.TelemetrySettings{}) + require.Error(t, err) +} + +func TestUnknownProductErrors(t *testing.T) { + cc := &captureConsumer{} + _, err := New(zap.NewNop(), Config{Workers: 1, Rate: time.Second, EnabledProducts: []string{"nope"}}, cc, embed.TelemetrySettings{}) + require.ErrorContains(t, err, "unknown f5 product") +} + +func TestAllTenProductsRegistered(t *testing.T) { + g, _ := newGen(t, DefaultConfig()) + require.Len(t, g.products, 10) +} + +func TestEnabledProductsFilter(t *testing.T) { + g, _ := newGen(t, Config{Workers: 1, Rate: time.Second, EnabledProducts: []string{"asm"}}) + require.Len(t, g.products, 1) + // Every built line must be an ASM line. + for i := 0; i < 50; i++ { + line := g.buildLine(rngFor(int64(i))) + require.Contains(t, line, " ASM: ") + } +} + +func TestDeterministicFromSeed(t *testing.T) { + g, _ := newGen(t, Config{Workers: 1, Rate: time.Second, Hostname: "bigip1"}) + a := g.buildLine(rngFor(99)) + b := g.buildLine(rngFor(99)) + require.Equal(t, a, b) +} + +func TestWeightsBiasSelection(t *testing.T) { + // Weight asm >> the rest; asm should dominate the mix. + g, _ := newGen(t, Config{Workers: 1, Rate: time.Second, Weights: map[string]float64{"asm": 1000}}) + asm := 0 + for i := 0; i < 500; i++ { + if strings.Contains(g.buildLine(rngFor(int64(i))), " ASM: ") { + asm++ + } + } + require.Greater(t, asm, 450, "asm weight 1000 should dominate") +} + +func TestStartStopEmits(t *testing.T) { + g, cc := newGen(t, Config{Workers: 2, Rate: 5 * time.Millisecond, Seed: 1}) + require.NoError(t, g.Start(context.Background())) + require.Eventually(t, func() bool { return cc.count() > 0 }, 2*time.Second, 10*time.Millisecond) + require.NoError(t, g.Stop(context.Background())) +} + +func TestSupportedTelemetry(t *testing.T) { + g, _ := newGen(t, DefaultConfig()) + require.Equal(t, []telemetry.Type{telemetry.Logs}, g.SupportedTelemetry()) +} diff --git a/generator/f5/products/afm/afm.go b/generator/f5/products/afm/afm.go new file mode 100644 index 00000000..9e1a9062 --- /dev/null +++ b/generator/f5/products/afm/afm.go @@ -0,0 +1,85 @@ +// Package afm registers the BIG-IP AFM (Advanced Firewall Manager) +// product: L3/L4 network firewall events. +// +// Byte-exact to F5's documented DEFAULT network-firewall log format (the +// "None" storage-format type): a comma-separated, double-quoted, positional +// value list in the documented field order below. +// +// Reference (validated 2026-09-25): +// +// https://techdocs.f5.com/kb/en-us/products/big-ip-afm/manuals/product/network-firewall-policies-implementations-11-6-0/13.html +// (Local Logging with the Network Firewall — default log format field order.) +package afm + +import ( + "fmt" + "math/rand" + "strings" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/internal/datagen" +) + +func init() { catalog.Register(catalog.Product{Name: "afm", Build: build}) } + +// defaultFields is the documented AFM default ("None") log format field order. +// The wire format is positional quoted CSV in exactly this order. +var defaultFields = []string{ + "management_ip_address", + "bigip_hostname", + "context_type", + "context_name", + "src_ip", + "dest_ip", + "src_port", + "dest_port", + "vlan", + "protocol", + "route_domain", + "acl_rule_name", + "action", + "drop_reason", +} + +// DefaultFields returns the documented default field order (for tests). +func DefaultFields() []string { return append([]string(nil), defaultFields...) } + +var ( + actions = []string{"Drop", "Reject", "Accept", "Accept-Decisively"} + protocols = []string{"tcp", "udp", "icmp"} + dropReasons = []string{"Policy", "Blacklisted address", "No route to host", "Port denied"} + rules = []string{"deny_inbound_rfc1918", "allow_web", "block_geo_cn", "default_deny"} + ctxTypes = []string{"Virtual Server", "Route Domain", "Global", "Self IP"} +) + +func build(r *rand.Rand, c *catalog.Ctx) string { + action := actions[r.Intn(len(actions))] + dropReason := "" + if action == "Drop" || action == "Reject" { + dropReason = dropReasons[r.Intn(len(dropReasons))] + } + // Positional values in defaultFields order. + vals := map[string]string{ + "management_ip_address": datagen.RandomPrivateIPv4(r), + "bigip_hostname": c.Hostname, + "context_type": ctxTypes[r.Intn(len(ctxTypes))], + "context_name": "/Common/vs_app", + "src_ip": datagen.RandomPublicIPv4(r), + "dest_ip": datagen.RandomPrivateIPv4(r), + "src_port": fmt.Sprintf("%d", 1024+r.Intn(64000)), + "dest_port": fmt.Sprintf("%d", []int{80, 443, 22, 53, 3389}[r.Intn(5)]), + "vlan": "/Common/external", + "protocol": protocols[r.Intn(len(protocols))], + "route_domain": "0", + "acl_rule_name": rules[r.Intn(len(rules))], + "action": action, + "drop_reason": dropReason, + } + + header := catalog.SyslogHeader(134, c.Now, c.Hostname, "tmm", 4000+r.Intn(2000)) + parts := make([]string, len(defaultFields)) + for i, name := range defaultFields { + parts[i] = `"` + vals[name] + `"` + } + return header + " " + strings.Join(parts, ",") +} diff --git a/generator/f5/products/afm/afm_test.go b/generator/f5/products/afm/afm_test.go new file mode 100644 index 00000000..4beaabdc --- /dev/null +++ b/generator/f5/products/afm/afm_test.go @@ -0,0 +1,44 @@ +package afm + +import ( + "math/rand" + "strings" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministic(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + got := build(rand.New(rand.NewSource(3)), c) + require.Equal(t, got, build(rand.New(rand.NewSource(3)), c)) + require.True(t, strings.HasPrefix(got, "<134>Sep 24 15:04:05 bigip1 tmm["), got) +} + +// TestDefaultFieldCountAndPositions asserts the emitted positional CSV matches +// the documented AFM default field order (14 fields). +func TestDefaultFieldCountAndPositions(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + got := build(rand.New(rand.NewSource(3)), c) + body := got[strings.Index(got, "]: ")+3:] + fields := strings.Split(body, ",") + + require.Len(t, DefaultFields(), 14) + require.Len(t, fields, 14, "emitted field count must match the documented default") + + // Every value is double-quoted. + for i, f := range fields { + require.True(t, strings.HasPrefix(f, `"`) && strings.HasSuffix(f, `"`), "field %d not quoted: %s", i, f) + } + // bigip_hostname is at index 1; context_name at 3; action at 12. + require.Equal(t, `"bigip1"`, fields[1]) + require.Equal(t, `"/Common/vs_app"`, fields[3]) + require.Contains(t, []string{`"Drop"`, `"Reject"`, `"Accept"`, `"Accept-Decisively"`}, fields[12]) +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("afm") + require.True(t, ok) +} diff --git a/generator/f5/products/apm/apm.go b/generator/f5/products/apm/apm.go new file mode 100644 index 00000000..1d04a065 --- /dev/null +++ b/generator/f5/products/apm/apm.go @@ -0,0 +1,60 @@ +// Package apm registers the BIG-IP APM (Access Policy Manager) product: +// access / authentication / SSO logs. +// +// APM logs are per-message-code free-text templates, not a fixed positional +// field set. This emits a realistic instance of the default APM access-log +// messages (MCP-style code + access-policy + session context). +// Config-dependent. +// +// Reference (validated 2026-09-25): +// +// https://techdocs.f5.com/en-us/bigip-17-5-0/big-ip-access-policy-manager-visual-policy-editor.html +package apm + +import ( + "fmt" + "math/rand" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/internal/datagen" +) + +func init() { catalog.Register(catalog.Product{Name: "apm", Build: build}) } + +var users = []string{"jsmith", "adoe", "svc_app", "contractor1", "admin"} + +type event struct { + code string + render func(user, session, clientIP string) string +} + +var events = []event{ + {"01490005", func(_, _, _ string) string { + return "Following rule 'fallback' from item 'Logon Page' to ending 'Allow'" + }}, + {"01490000", func(_, session, clientIP string) string { + return fmt.Sprintf("Session %s created from client %s", session, clientIP) + }}, + {"01490102", func(user, _, _ string) string { + return fmt.Sprintf("Access policy result: LTM+APM_Mode for user %s", user) + }}, + {"01490547", func(user, _, _ string) string { + return fmt.Sprintf("SSO: successful Kerberos SSO for user %s", user) + }}, + {"01490567", func(user, _, _ string) string { + return fmt.Sprintf("Session deleted due to user logout for %s", user) + }}, + {"0149004b", func(user, _, _ string) string { + return fmt.Sprintf("Authentication failed for user %s (Active Directory)", user) + }}, +} + +func build(r *rand.Rand, c *catalog.Ctx) string { + header := catalog.SyslogHeader(134, c.Now, c.Hostname, "apmd", 4000+r.Intn(2000)) + ev := events[r.Intn(len(events))] + user := users[r.Intn(len(users))] + session := fmt.Sprintf("%08x", r.Uint32()) + msg := ev.render(user, session, datagen.RandomPublicIPv4(r)) + // APM: ":5: /Common/:Common:: " + return fmt.Sprintf("%s %s:5: /Common/access_policy:Common:%s: %s", header, ev.code, session, msg) +} diff --git a/generator/f5/products/apm/apm_test.go b/generator/f5/products/apm/apm_test.go new file mode 100644 index 00000000..d154aed9 --- /dev/null +++ b/generator/f5/products/apm/apm_test.go @@ -0,0 +1,25 @@ +package apm + +import ( + "math/rand" + "strings" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministicAndShaped(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + got := build(rand.New(rand.NewSource(9)), c) + require.Equal(t, got, build(rand.New(rand.NewSource(9)), c)) + + require.True(t, strings.HasPrefix(got, "<134>Sep 24 15:04:05 bigip1 apmd["), got) + require.Contains(t, got, ":5: /Common/access_policy:Common:") +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("apm") + require.True(t, ok) +} diff --git a/generator/f5/products/appprotect/appprotect.go b/generator/f5/products/appprotect/appprotect.go new file mode 100644 index 00000000..df601a8a --- /dev/null +++ b/generator/f5/products/appprotect/appprotect.go @@ -0,0 +1,156 @@ +// Package appprotect registers the NGINX App Protect (WAF on NGINX) +// product: security events in the "default" security-log format. +// +// The field set and order are the documented "default" predefined format +// from F5's NGINX App Protect security-log reference — the comma-separated +// key="value" attributes the default format emits, in the documented order. +// +// Reference (validated 2026-09-25): +// +// https://docs.nginx.com/waf/logging/security-logs/ +// (NGINX App Protect WAF Security Log — "Available Security Log +// Attributes", attributes included in the `default` format.) +package appprotect + +import ( + "fmt" + "math/rand" + "strings" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/internal/datagen" +) + +func init() { catalog.Register(catalog.Product{Name: "nginx-app-protect", Build: build}) } + +// defaultFields is the ordered attribute set of the App Protect "default" +// security-log format, exactly as published in the security-log reference. +// The order here is the wire order build() emits. +var defaultFields = []string{ + "attack_type", + "blocking_exception_reason", + "bot_anomalies", + "bot_category", + "bot_signature_name", + "client_class", + "date_time", + "dest_ip", + "dest_port", + "enforced_bot_anomalies", + "ip_client", + "is_truncated_bool", + "json_log", + "method", + "outcome", + "outcome_reason", + "policy_name", + "protocol", + "request", + "request_status", + "response_code", + "severity", + "sig_cves", + "sig_ids", + "sig_names", + "sig_set_names", + "src_port", + "sub_violations", + "support_id", + "threat_campaign_names", + "unit_hostname", + "uri", + "username", + "violation_details", + "violation_rating", + "violations", + "vs_name", + "x_forwarded_for_header_value", + "transport_protocol", + "client_application", + "client_application_version", +} + +// DefaultFields returns the documented default-format attribute order. Exposed +// for tests that assert the emitted field set/order matches the spec. +func DefaultFields() []string { return append([]string(nil), defaultFields...) } + +var ( + attacks = []string{"Non-browser Client", "SQL-Injection", "Cross Site Scripting (XSS)", "Abuse of Functionality"} + sigNames = []string{"XSS script tag end (Parameter)", "SQL-INJ UNION SELECT", "Automated client (cookie header)"} + severities = []string{"Critical", "Error", "Warning"} + statuses = []string{"blocked", "alerted", "passed"} + methods = []string{"GET", "POST", "PUT"} + policies = []string{"app_protect_default_policy", "strict_owasp", "api_policy"} + botCats = []string{"N/A", "Untrusted Bot", "Trusted Bot", "Malicious Bot"} + clientClas = []string{"Untrusted Bot", "Browser", "Trusted Bot", "Suspicious Browser"} +) + +func build(r *rand.Rand, c *catalog.Ctx) string { + status := statuses[r.Intn(len(statuses))] + outcome, outcomeReason, respCode := "PASSED", "SECURITY_WAF_OK", 200 + if status == "blocked" { + outcome, outcomeReason, respCode = "REJECTED", "SECURITY_WAF_VIOLATION", 0 + } + attack := attacks[r.Intn(len(attacks))] + + // vals maps each documented attribute to its emitted value. Optional + // attributes with no value emit empty per spec (e.g. sig_cves="N/A"). + vals := map[string]string{ + "attack_type": attack, + "blocking_exception_reason": "N/A", + "bot_anomalies": "N/A", + "bot_category": botCats[r.Intn(len(botCats))], + "bot_signature_name": "N/A", + "client_class": clientClas[r.Intn(len(clientClas))], + "date_time": c.Now.Format("2006-01-02 15:04:05"), + "dest_ip": datagen.RandomPrivateIPv4(r), + "dest_port": "443", + "enforced_bot_anomalies": "N/A", + "ip_client": datagen.RandomPublicIPv4(r), + "is_truncated_bool": "false", + "json_log": "N/A", + "method": methods[r.Intn(len(methods))], + "outcome": outcome, + "outcome_reason": outcomeReason, + "policy_name": policies[r.Intn(len(policies))], + "protocol": "HTTPS", + "request": "GET /api/v1/orders HTTP/1.1", + "request_status": status, + "response_code": fmt.Sprintf("%d", respCode), + "severity": severities[r.Intn(len(severities))], + "sig_cves": "N/A", + "sig_ids": fmt.Sprintf("%d", 200000000+r.Intn(99999999)), + "sig_names": sigNames[r.Intn(len(sigNames))], + "sig_set_names": "{Automated Threats;High Accuracy Signatures}", + "src_port": fmt.Sprintf("%d", 1024+r.Intn(64000)), + "sub_violations": "N/A", + "support_id": fmt.Sprintf("%d", 1000000000000000+r.Int63n(8999999999999999)), + "threat_campaign_names": "N/A", + "unit_hostname": c.Hostname, + "uri": "/api/v1/orders", + "username": "N/A", + "violation_details": "", + "violation_rating": fmt.Sprintf("%d", 1+r.Intn(5)), + "violations": "Illegal meta character in value", + "vs_name": "/Common/app.example.com", + "x_forwarded_for_header_value": datagen.RandomPublicIPv4(r), + "transport_protocol": "TCP", + "client_application": "N/A", + "client_application_version": "N/A", + } + + header := catalog.SyslogHeader(134, c.Now, c.Hostname, "app_protect", 0) + var b strings.Builder + b.WriteString(header) + b.WriteByte(' ') + for i, name := range defaultFields { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(name) + b.WriteString(`="`) + b.WriteString(vals[name]) + b.WriteByte('"') + } + return b.String() +} diff --git a/generator/f5/products/appprotect/appprotect_test.go b/generator/f5/products/appprotect/appprotect_test.go new file mode 100644 index 00000000..1f976da1 --- /dev/null +++ b/generator/f5/products/appprotect/appprotect_test.go @@ -0,0 +1,47 @@ +package appprotect + +import ( + "math/rand" + "regexp" + "strings" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministic(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "nap1"} + got := build(rand.New(rand.NewSource(13)), c) + require.Equal(t, got, build(rand.New(rand.NewSource(13)), c)) + require.True(t, strings.HasPrefix(got, "<134>Sep 24 15:04:05 nap1 app_protect:"), got) +} + +// TestDefaultFieldSetAndOrder asserts the emitted key set and order exactly +// match the documented App Protect "default" security-log attribute list. +func TestDefaultFieldSetAndOrder(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "nap1"} + got := build(rand.New(rand.NewSource(1)), c) + + // Strip the syslog header, keep the key="value",... body. + body := got[strings.Index(got, "app_protect:")+len("app_protect:")+1:] + + // Anchor to field boundaries (start-of-body or a comma) so a `key="` + // substring inside a value (e.g. version="1.0" in violation_details) is + // not mistaken for a field. + keyRe := regexp.MustCompile(`(?:^|,)([a-z0-9_]+)="`) + matches := keyRe.FindAllStringSubmatch(body, -1) + keys := make([]string, 0, len(matches)) + for _, m := range matches { + keys = append(keys, m[1]) + } + + require.Equal(t, DefaultFields(), keys, "emitted field set/order must match the documented default format") + require.Len(t, keys, 41) +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("nginx-app-protect") + require.True(t, ok) +} diff --git a/generator/f5/products/asm/asm.go b/generator/f5/products/asm/asm.go new file mode 100644 index 00000000..237940b5 --- /dev/null +++ b/generator/f5/products/asm/asm.go @@ -0,0 +1,89 @@ +// Package asm registers the BIG-IP ASM (Application Security Manager) +// product: WAF security events / attack signatures. +// +// The ASM remote-logging Storage Format is operator-configurable (Field-List / +// User-Defined), so there is no single positional wire spec. This models F5's +// documented DEFAULT syslog field set, in the documented order, emitted as +// comma-separated key="value" pairs. The full field catalog is GUI-only and not +// published as an ordered reference. +// +// Reference (validated 2026-09-25): +// +// https://techdocs.f5.com/en-us/bigip-17-5-0/big-ip-asm-implementations/logging-application-security-events.html +// (Logging Application Security Events — default syslog format field set.) +package asm + +import ( + "fmt" + "math/rand" + "strings" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/internal/datagen" +) + +func init() { catalog.Register(catalog.Product{Name: "asm", Build: build}) } + +// defaultFields is F5's documented default ASM syslog field set, in order. +var defaultFields = []string{ + "rejection_description", + "request_violation", + "support_id", + "source_ip", + "xff_ip", + "source_port", + "destination_ip", + "destination_port", + "route_domain", + "http_classifier", + "scheme", + "geographic_location", + "request", + "username", + "session_id", + "violation_rating", +} + +// DefaultFields returns the documented default field order (for tests). +func DefaultFields() []string { return append([]string(nil), defaultFields...) } + +var ( + violations = []string{"Attack signature detected", "Illegal meta character in value", "Illegal URL length", "Illegal HTTP method"} + statuses = []string{"blocked", "alerted", "passed"} + classes = []string{"/Common/prod_waf_policy", "/Common/api_protection", "/Common/owasp_top10"} + geos = []string{"US", "GB", "DE", "CN"} + schemes = []string{"https", "http"} +) + +func build(r *rand.Rand, c *catalog.Ctx) string { + status := statuses[r.Intn(len(statuses))] + rejection := "N/A" + if status == "blocked" { + rejection = "Request was blocked" + } + vals := map[string]string{ + "rejection_description": rejection, + "request_violation": violations[r.Intn(len(violations))], + "support_id": fmt.Sprintf("%d", 1000000000000000+r.Int63n(8999999999999999)), + "source_ip": datagen.RandomPublicIPv4(r), + "xff_ip": datagen.RandomPublicIPv4(r), + "source_port": fmt.Sprintf("%d", 1024+r.Intn(64000)), + "destination_ip": datagen.RandomPrivateIPv4(r), + "destination_port": "443", + "route_domain": "0", + "http_classifier": classes[r.Intn(len(classes))], + "scheme": schemes[r.Intn(len(schemes))], + "geographic_location": geos[r.Intn(len(geos))], + "request": "GET /api/v1/orders?id=1%27%20or%20%271%27=%271 HTTP/1.1", + "username": "N/A", + "session_id": fmt.Sprintf("%x", r.Uint64()), + "violation_rating": fmt.Sprintf("%d", 1+r.Intn(5)), + } + + header := catalog.SyslogHeader(134, c.Now, c.Hostname, "ASM", 0) + parts := make([]string, len(defaultFields)) + for i, name := range defaultFields { + parts[i] = name + `="` + vals[name] + `"` + } + return header + " " + strings.Join(parts, ",") +} diff --git a/generator/f5/products/asm/asm_test.go b/generator/f5/products/asm/asm_test.go new file mode 100644 index 00000000..965b3aa0 --- /dev/null +++ b/generator/f5/products/asm/asm_test.go @@ -0,0 +1,41 @@ +package asm + +import ( + "math/rand" + "regexp" + "strings" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministic(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + got := build(rand.New(rand.NewSource(7)), c) + require.Equal(t, got, build(rand.New(rand.NewSource(7)), c)) + require.True(t, strings.HasPrefix(got, "<134>Sep 24 15:04:05 bigip1 ASM:"), got) +} + +// TestDefaultFieldSetAndOrder asserts the emitted key set and order match F5's +// documented default ASM syslog field set (16 fields). +func TestDefaultFieldSetAndOrder(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + got := build(rand.New(rand.NewSource(1)), c) + body := got[strings.Index(got, "ASM:")+len("ASM:")+1:] + + keyRe := regexp.MustCompile(`(?:^|,)([a-z0-9_]+)="`) + matches := keyRe.FindAllStringSubmatch(body, -1) + keys := make([]string, 0, len(matches)) + for _, m := range matches { + keys = append(keys, m[1]) + } + require.Equal(t, DefaultFields(), keys) + require.Len(t, keys, 16) +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("asm") + require.True(t, ok) +} diff --git a/generator/f5/products/audit/audit.go b/generator/f5/products/audit/audit.go new file mode 100644 index 00000000..130de4f9 --- /dev/null +++ b/generator/f5/products/audit/audit.go @@ -0,0 +1,48 @@ +// Package audit registers the BIG-IP audit product: configuration audit +// records emitted by mcpd across modules. +// +// Byte-exact to the documented mcpd AUDIT message template: +// +// :: AUDIT - client , user - transaction #- - +// +// Reference (validated 2026-09-25): +// +// https://techdocs.f5.com/en-us/bigip-17-5-0/external-monitoring-of-big-ip-systems-implementations.html +// (BIG-IP audit logging — mcpd AUDIT record format.) +package audit + +import ( + "fmt" + "math/rand" + + "github.com/observiq/blitz/generator/f5/catalog" +) + +func init() { catalog.Register(catalog.Product{Name: "audit", Build: build}) } + +var ( + clients = []string{"tmsh", "GUI", "iControl REST", "httpd(mod_auth_pam)"} + users = []string{"admin", "operator", "svc_automation", "auditor"} + cmds = []string{"create", "modify", "delete", "list"} + objects = []string{ + "ltm virtual /Common/vs_https_443", + "ltm pool /Common/pool_web", + "security firewall policy /Common/fw_policy", + "auth user operator", + "sys ntp", + } + codes = []string{"01070417", "01071031", "01070734", "01420002"} +) + +func build(r *rand.Rand, c *catalog.Ctx) string { + header := catalog.SyslogHeader(133, c.Now, c.Hostname, "mcpd", 4000+r.Intn(2000)) + code := codes[r.Intn(len(codes))] + client := clients[r.Intn(len(clients))] + user := users[r.Intn(len(users))] + cmd := cmds[r.Intn(len(cmds))] + object := objects[r.Intn(len(objects))] + txn := 1000 + r.Intn(90000) + // Documented mcpd AUDIT template. + return fmt.Sprintf("%s %s:5: AUDIT - client %s, user %s - transaction #%d-1 - %s %s", + header, code, client, user, txn, cmd, object) +} diff --git a/generator/f5/products/audit/audit_test.go b/generator/f5/products/audit/audit_test.go new file mode 100644 index 00000000..51786ff1 --- /dev/null +++ b/generator/f5/products/audit/audit_test.go @@ -0,0 +1,32 @@ +package audit + +import ( + "math/rand" + "regexp" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministic(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + got := build(rand.New(rand.NewSource(5)), c) + require.Equal(t, got, build(rand.New(rand.NewSource(5)), c)) +} + +// TestAuditTemplateStructure asserts the emitted line matches the documented +// mcpd AUDIT template segment order. +func TestAuditTemplateStructure(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + got := build(rand.New(rand.NewSource(5)), c) + re := regexp.MustCompile(`^<133>Sep 24 15:04:05 bigip1 mcpd\[\d+\]: ` + + `\d{8}:5: AUDIT - client [^,]+, user \S+ - transaction #\d+-1 - (create|modify|delete|list) .+$`) + require.Regexp(t, re, got) +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("audit") + require.True(t, ok) +} diff --git a/generator/f5/products/dns/dns.go b/generator/f5/products/dns/dns.go new file mode 100644 index 00000000..6614f1fb --- /dev/null +++ b/generator/f5/products/dns/dns.go @@ -0,0 +1,42 @@ +// Package dns registers the BIG-IP DNS (formerly GTM) product: GSLB / +// DNS query-response logs from the DNS data plane. +// +// BIG-IP DNS logging is driven by an operator-defined DNS logging profile, +// so there is no fixed positional wire spec. This emits a realistic instance +// of a default DNS/GTM query-resolution line. Config-dependent. +// +// Reference (validated 2026-09-25): +// +// https://techdocs.f5.com/en-us/bigip-17-5-0/external-monitoring-of-big-ip-systems-implementations.html +package dns + +import ( + "fmt" + "math/rand" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/internal/datagen" +) + +func init() { catalog.Register(catalog.Product{Name: "dns", Build: build}) } + +var ( + qnames = []string{"www.example.com", "api.example.com", "app.corp.example.net", "mail.example.org"} + qtypes = []string{"A", "AAAA", "CNAME", "MX", "SRV"} + wideips = []string{"/Common/www_gslb", "/Common/api_gslb", "/Common/app_gslb"} + pools = []string{"/Common/pool_us_east", "/Common/pool_eu_west", "/Common/pool_ap_south"} + results = []string{"RESOLVED", "NOERROR", "NXDOMAIN", "SERVFAIL"} +) + +func build(r *rand.Rand, c *catalog.Ctx) string { + header := catalog.SyslogHeader(134, c.Now, c.Hostname, "tmm", 4000+r.Intn(2000)) + qname := qnames[r.Intn(len(qnames))] + qtype := qtypes[r.Intn(len(qtypes))] + wideip := wideips[r.Intn(len(wideips))] + pool := pools[r.Intn(len(pools))] + member := datagen.RandomPublicIPv4(r) + result := results[r.Intn(len(results))] + // GTM/DNS query-resolution line. + return fmt.Sprintf(`%s client %s#%d: query [%s %s] wideip %s -> pool %s member %s result %s rtt %dms`, + header, datagen.RandomPublicIPv4(r), 1024+r.Intn(64000), qname, qtype, wideip, pool, member, result, 1+r.Intn(200)) +} diff --git a/generator/f5/products/dns/dns_test.go b/generator/f5/products/dns/dns_test.go new file mode 100644 index 00000000..e32348d1 --- /dev/null +++ b/generator/f5/products/dns/dns_test.go @@ -0,0 +1,26 @@ +package dns + +import ( + "math/rand" + "strings" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministicAndShaped(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + got := build(rand.New(rand.NewSource(11)), c) + require.Equal(t, got, build(rand.New(rand.NewSource(11)), c)) + + require.True(t, strings.HasPrefix(got, "<134>Sep 24 15:04:05 bigip1 tmm["), got) + require.Contains(t, got, "wideip /Common/") + require.Contains(t, got, "query [") +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("dns") + require.True(t, ok) +} diff --git a/generator/f5/products/f5os/f5os.go b/generator/f5/products/f5os/f5os.go new file mode 100644 index 00000000..a67b5a56 --- /dev/null +++ b/generator/f5/products/f5os/f5os.go @@ -0,0 +1,57 @@ +// Package f5os registers the F5OS / TMOS platform product: platform- +// level audit and system events (chassis, tenant, service lifecycle). +// +// F5OS platform logs come from several daemons (confd, systemd, platform) +// with no single positional wire spec. This emits realistic instances of +// default platform audit/system log lines. Config-dependent. +// +// Reference (validated 2026-09-25): +// +// https://techdocs.f5.com/en-us/f5os-a-1-8-0/f5-rseries-systems-administration-configuration.html +package f5os + +import ( + "fmt" + "math/rand" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/internal/datagen" +) + +func init() { catalog.Register(catalog.Product{Name: "f5os", Build: build}) } + +var users = []string{"admin", "root", "svc_orchestration"} + +type ev struct { + tag string + pri int + render func(r *rand.Rand, user, ip string) string +} + +var events = []ev{ + {"confd", 133, func(_ *rand.Rand, user, ip string) string { + return fmt.Sprintf("audit - user: %s from %s - command: set tenant tenant1 config state deployed", user, ip) + }}, + {"systemd", 150, func(_ *rand.Rand, _, _ string) string { + return "Started F5 platform tenant tenant1.service" + }}, + {"platform", 147, func(r *rand.Rand, _, _ string) string { + return fmt.Sprintf("chassis partition 1 blade 1: temperature nominal (%dC)", 30+r.Intn(15)) + }}, + {"velos-controller", 148, func(_ *rand.Rand, _, _ string) string { + return "tenant tenant1 transitioned Running -> Deployed" + }}, + {"sshd", 134, func(r *rand.Rand, user, ip string) string { + return fmt.Sprintf("Accepted publickey for %s from %s port %d ssh2", user, ip, 1024+r.Intn(64000)) + }}, + {"confd", 132, func(_ *rand.Rand, user, ip string) string { + return fmt.Sprintf("audit - user: %s from %s - command: delete interfaces interface 2.0", user, ip) + }}, +} + +func build(r *rand.Rand, c *catalog.Ctx) string { + e := events[r.Intn(len(events))] + header := catalog.SyslogHeader(e.pri, c.Now, c.Hostname, e.tag, 1000+r.Intn(9000)) + msg := e.render(r, users[r.Intn(len(users))], datagen.RandomPrivateIPv4(r)) + return header + " " + msg +} diff --git a/generator/f5/products/f5os/f5os_test.go b/generator/f5/products/f5os/f5os_test.go new file mode 100644 index 00000000..2d64f477 --- /dev/null +++ b/generator/f5/products/f5os/f5os_test.go @@ -0,0 +1,24 @@ +package f5os + +import ( + "math/rand" + "strings" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministicAndShaped(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "f5os-a"} + got := build(rand.New(rand.NewSource(23)), c) + require.Equal(t, got, build(rand.New(rand.NewSource(23)), c)) + require.True(t, strings.HasPrefix(got, "<"), got) + require.Contains(t, got, "f5os-a") +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("f5os") + require.True(t, ok) +} diff --git a/generator/f5/products/irules/irules.go b/generator/f5/products/irules/irules.go new file mode 100644 index 00000000..fc9baeaf --- /dev/null +++ b/generator/f5/products/irules/irules.go @@ -0,0 +1,57 @@ +// Package irules registers the iRules logging product: operator-emitted +// log lines from custom TCL rules. +// +// iRules log output is fully operator-defined (arbitrary TCL `log` statements), +// so no wire spec exists. This emits realistic instances of common operator +// log lines in the tmm "Rule /Common/ :" framing. Config-dependent. +// +// Reference (validated 2026-09-25): +// +// https://clouddocs.f5.com/api/irules/log.html +package irules + +import ( + "fmt" + "math/rand" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/internal/datagen" +) + +func init() { catalog.Register(catalog.Product{Name: "irules", Build: build}) } + +var rules = []string{"log_http_requests", "maintenance_page", "block_bad_bots", "header_insert", "rate_limit"} + +type ev struct { + name string + render func(clientIP string, port int) string +} + +var events = []ev{ + {"HTTP_REQUEST", func(ip string, port int) string { + return fmt.Sprintf("client %s:%d requested /login, inserting X-Forwarded-For", ip, port) + }}, + {"HTTP_REQUEST", func(ip string, _ int) string { + return fmt.Sprintf("blocked bad bot from %s (User-Agent matched)", ip) + }}, + {"CLIENTSSL_HANDSHAKE", func(ip string, port int) string { + return fmt.Sprintf("TLS handshake complete for %s:%d, cipher ECDHE-RSA-AES256-GCM-SHA384", ip, port) + }}, + {"LB_SELECTED", func(ip string, _ int) string { + return fmt.Sprintf("selected pool member for %s", ip) + }}, + {"HTTP_RESPONSE", func(ip string, _ int) string { + return fmt.Sprintf("serving maintenance page to %s", ip) + }}, + {"RULE_INIT", func(_ string, _ int) string { + return "rule initialized" + }}, +} + +func build(r *rand.Rand, c *catalog.Ctx) string { + header := catalog.SyslogHeader(134, c.Now, c.Hostname, "tmm", 4000+r.Intn(2000)) + rule := rules[r.Intn(len(rules))] + e := events[r.Intn(len(events))] + msg := e.render(datagen.RandomPublicIPv4(r), 1024+r.Intn(64000)) + return fmt.Sprintf("%s Rule /Common/%s <%s>: %s", header, rule, e.name, msg) +} diff --git a/generator/f5/products/irules/irules_test.go b/generator/f5/products/irules/irules_test.go new file mode 100644 index 00000000..f1c79428 --- /dev/null +++ b/generator/f5/products/irules/irules_test.go @@ -0,0 +1,26 @@ +package irules + +import ( + "math/rand" + "strings" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministicAndShaped(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + got := build(rand.New(rand.NewSource(17)), c) + require.Equal(t, got, build(rand.New(rand.NewSource(17)), c)) + + require.True(t, strings.HasPrefix(got, "<134>Sep 24 15:04:05 bigip1 tmm["), got) + require.Contains(t, got, "Rule /Common/") + require.Contains(t, got, ">:") +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("irules") + require.True(t, ok) +} diff --git a/generator/f5/products/ltm/ltm.go b/generator/f5/products/ltm/ltm.go new file mode 100644 index 00000000..56aba666 --- /dev/null +++ b/generator/f5/products/ltm/ltm.go @@ -0,0 +1,49 @@ +// Package ltm registers the BIG-IP LTM (Local Traffic Manager) product: +// request logs from the TMM data plane. +// +// LTM request logging uses an operator-defined Request Logging profile +// format string, so there is no fixed positional wire spec. This emits a +// realistic instance of a common default request-logging profile +// (combined-style with virtual-server/pool context). Config-dependent. +// +// Reference (validated 2026-09-25): +// +// https://techdocs.f5.com/en-us/bigip-17-5-0/big-ip-ltm-implementations/configuring-request-logging.html +package ltm + +import ( + "fmt" + "math/rand" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/internal/datagen" +) + +func init() { catalog.Register(catalog.Product{Name: "ltm", Build: build}) } + +var methods = []string{"GET", "POST", "PUT", "DELETE", "HEAD"} +var uris = []string{"/", "/index.html", "/api/v1/orders", "/login", "/static/app.js", "/health", "/cart/checkout"} +var statuses = []int{200, 200, 200, 301, 302, 404, 500, 503} +var vips = []string{"/Common/vs_https_443", "/Common/vs_http_80", "/Common/vs_api_8443"} +var agents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "curl/8.4.0", + "python-requests/2.31.0", +} + +func build(r *rand.Rand, c *catalog.Ctx) string { + client := datagen.RandomPublicIPv4(r) + vip := vips[r.Intn(len(vips))] + method := methods[r.Intn(len(methods))] + uri := uris[r.Intn(len(uris))] + status := statuses[r.Intn(len(statuses))] + bytes := 200 + r.Intn(50000) + ua := agents[r.Intn(len(agents))] + pool := "/Common/pool_web" + member := fmt.Sprintf("%s:%d", datagen.RandomPrivateIPv4(r), []int{80, 443, 8080}[r.Intn(3)]) + + header := catalog.SyslogHeader(134, c.Now, c.Hostname, "tmm", 4000+r.Intn(2000)) + // F5 LTM request-logging profile: combined-log body plus virtual/pool context. + return fmt.Sprintf(`%s %s - - [%s] "%s %s HTTP/1.1" %d %d "-" "%s" vs=%s pool=%s member=%s`, + header, client, c.Now.Format("02/Jan/2006:15:04:05 -0700"), method, uri, status, bytes, ua, vip, pool, member) +} diff --git a/generator/f5/products/ltm/ltm_test.go b/generator/f5/products/ltm/ltm_test.go new file mode 100644 index 00000000..f8d72e6c --- /dev/null +++ b/generator/f5/products/ltm/ltm_test.go @@ -0,0 +1,28 @@ +package ltm + +import ( + "math/rand" + "strings" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministicAndShaped(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "bigip1"} + + got := build(rand.New(rand.NewSource(42)), c) + again := build(rand.New(rand.NewSource(42)), c) + require.Equal(t, got, again, "same seed must yield identical output") + + require.True(t, strings.HasPrefix(got, "<134>Sep 24 15:04:05 bigip1 tmm["), got) + require.Contains(t, got, "vs=/Common/") + require.Contains(t, got, "HTTP/1.1") +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("ltm") + require.True(t, ok) +} diff --git a/generator/f5/products/nginxplus/nginxplus.go b/generator/f5/products/nginxplus/nginxplus.go new file mode 100644 index 00000000..a2baf330 --- /dev/null +++ b/generator/f5/products/nginxplus/nginxplus.go @@ -0,0 +1,93 @@ +// Package nginxplus registers the NGINX-on-F5 (NGINX Plus) product: +// access and error logs. +// +// The access log is byte-exact to NGINX's documented DEFAULT "combined" +// log_format, extended with the documented NGINX Plus upstream timing +// variables (request_time + upstream_connect_time / upstream_header_time / +// upstream_response_time / upstream_addr) that distinguish Plus from the +// community nginx generator. The error log follows NGINX's default error_log +// line format. +// +// References (validated 2026-09-25): +// +// https://docs.nginx.com/nginx/admin-guide/monitoring/logging/ +// (combined default format; $upstream_*/$request_time timing variables.) +package nginxplus + +import ( + "fmt" + "math/rand" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/observiq/blitz/internal/datagen" +) + +func init() { catalog.Register(catalog.Product{Name: "nginx-plus", Build: build}) } + +// combinedVars is the ordered variable list of NGINX's default "combined" +// log_format: +// +// $remote_addr - $remote_user [$time_local] "$request" $status +// $body_bytes_sent "$http_referer" "$http_user_agent" +// +// plusExtraVars are the documented NGINX Plus upstream timing fields appended. +var ( + combinedVars = []string{ + "remote_addr", "remote_user", "time_local", "request", + "status", "body_bytes_sent", "http_referer", "http_user_agent", + } + plusExtraVars = []string{"request_time", "upstream_connect_time", "upstream_header_time", "upstream_response_time", "upstream_addr"} +) + +// AccessFormatVars returns the full ordered variable list of the access line +// (combined + Plus extras), for tests. +func AccessFormatVars() []string { + return append(append([]string(nil), combinedVars...), plusExtraVars...) +} + +var ( + methods = []string{"GET", "POST", "PUT", "DELETE"} + uris = []string{"/", "/api/v2/users", "/assets/main.css", "/health", "/checkout"} + statuses = []int{200, 200, 201, 301, 404, 502, 504} + agents = []string{"Mozilla/5.0", "curl/8.4.0", "kube-probe/1.29"} + errLevels = []string{"error", "warn", "crit"} + errReasons = []string{ + "upstream timed out (110: Connection timed out) while reading response header from upstream", + "connect() failed (111: Connection refused) while connecting to upstream", + "no live upstreams while connecting to upstream", + } +) + +func build(r *rand.Rand, c *catalog.Ctx) string { + if r.Intn(10) == 0 { // ~10% error log + return errorLine(r, c) + } + return accessLine(r, c) +} + +// accessLine emits the combined format followed by the Plus upstream fields. +func accessLine(r *rand.Rand, c *catalog.Ctx) string { + header := catalog.SyslogHeader(158, c.Now, c.Hostname, "nginx", 0) + remoteAddr := datagen.RandomPublicIPv4(r) + request := fmt.Sprintf("%s %s HTTP/1.1", methods[r.Intn(len(methods))], uris[r.Intn(len(uris))]) + status := statuses[r.Intn(len(statuses))] + bodyBytes := 100 + r.Intn(40000) + ua := agents[r.Intn(len(agents))] + upstreamAddr := fmt.Sprintf("%s:%d", datagen.RandomPrivateIPv4(r), []int{8080, 8443, 9000}[r.Intn(3)]) + + // combined: $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" + combined := fmt.Sprintf(`%s - - [%s] "%s" %d %d "-" "%s"`, + remoteAddr, c.Now.Format("02/Jan/2006:15:04:05 -0700"), request, status, bodyBytes, ua) + // Plus upstream timing extension. + plus := fmt.Sprintf(`rt=%.3f uct="%.3f" uht="%.3f" urt="%.3f" upstream_addr=%s`, + r.Float64(), r.Float64()/10, r.Float64()/5, r.Float64(), upstreamAddr) + return header + " " + combined + " " + plus +} + +func errorLine(r *rand.Rand, c *catalog.Ctx) string { + header := catalog.SyslogHeader(155, c.Now, c.Hostname, "nginx", 0) + // Default error_log line: [] #: * , client: ..., server: ..., request: ..., upstream: ... + return fmt.Sprintf(`%s %s [%s] %d#%d: *%d %s, client: %s, server: app.example.com, request: "GET / HTTP/1.1", upstream: "http://%s:8080/"`, + header, c.Now.Format("2006/01/02 15:04:05"), errLevels[r.Intn(len(errLevels))], 1000+r.Intn(9000), r.Intn(64), + r.Intn(1000000), errReasons[r.Intn(len(errReasons))], datagen.RandomPublicIPv4(r), datagen.RandomPrivateIPv4(r)) +} diff --git a/generator/f5/products/nginxplus/nginxplus_test.go b/generator/f5/products/nginxplus/nginxplus_test.go new file mode 100644 index 00000000..4cd65501 --- /dev/null +++ b/generator/f5/products/nginxplus/nginxplus_test.go @@ -0,0 +1,53 @@ +package nginxplus + +import ( + "math/rand" + "regexp" + "testing" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + "github.com/stretchr/testify/require" +) + +func TestBuildDeterministic(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "nginxp1"} + got := build(rand.New(rand.NewSource(2)), c) + require.Equal(t, got, build(rand.New(rand.NewSource(2)), c)) +} + +// TestAccessLineMatchesCombinedPlus asserts the access line is byte-exact to the +// combined format followed by the documented Plus upstream fields. +func TestAccessLineMatchesCombinedPlus(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "nginxp1"} + got := accessLine(rand.New(rand.NewSource(2)), c) + + // header + combined + plus. Verify the combined structure exactly, then the + // Plus extension fields in order. + re := regexp.MustCompile(`^<\d+>[^ ]+ [ \d]?\d \d\d:\d\d:\d\d nginxp1 nginx: ` + + `\S+ - - \[[^\]]+\] "[A-Z]+ \S+ HTTP/1\.1" \d{3} \d+ "-" "[^"]*" ` + + `rt=\d+\.\d{3} uct="\d+\.\d{3}" uht="\d+\.\d{3}" urt="\d+\.\d{3}" upstream_addr=\S+$`) + require.Regexp(t, re, got) +} + +func TestAccessFormatVarsOrder(t *testing.T) { + vars := AccessFormatVars() + require.Equal(t, []string{ + "remote_addr", "remote_user", "time_local", "request", "status", + "body_bytes_sent", "http_referer", "http_user_agent", + "request_time", "upstream_connect_time", "upstream_header_time", + "upstream_response_time", "upstream_addr", + }, vars) +} + +func TestErrorLineShaped(t *testing.T) { + c := &catalog.Ctx{Now: time.Date(2026, 9, 24, 15, 4, 5, 0, time.UTC), Hostname: "nginxp1"} + got := errorLine(rand.New(rand.NewSource(2)), c) + require.Contains(t, got, "client:") + require.Contains(t, got, "upstream:") +} + +func TestRegistered(t *testing.T) { + _, ok := catalog.Get("nginx-plus") + require.True(t, ok) +} diff --git a/internal/config/generator.go b/internal/config/generator.go index 84e007cf..108cd5bc 100644 --- a/internal/config/generator.go +++ b/internal/config/generator.go @@ -41,6 +41,8 @@ const ( // GeneratorTypeFIX represents the FIX (Financial Information // eXchange) protocol generator GeneratorTypeFIX GeneratorType = "fix" + // GeneratorTypeF5 represents the multi-product F5 log generator + GeneratorTypeF5 GeneratorType = "f5" ) // Generator contains configuration for log generators @@ -79,6 +81,8 @@ type Generator struct { Wel WelGeneratorConfig `yaml:"wel,omitempty" mapstructure:"wel,omitempty"` // FIX contains FIX generator configuration FIX FIXGeneratorConfig `yaml:"fix,omitempty" mapstructure:"fix,omitempty"` + // F5 contains F5 multi-product generator configuration + F5 F5GeneratorConfig `yaml:"f5,omitempty" mapstructure:"f5,omitempty"` } // Validate validates the generator configuration @@ -155,8 +159,12 @@ func (g *Generator) Validate() error { if err := g.FIX.Validate(); err != nil { return fmt.Errorf("fix generator validation failed: %w", err) } + case GeneratorTypeF5: + if err := g.F5.Validate(); err != nil { + return fmt.Errorf("f5 generator validation failed: %w", err) + } default: - return fmt.Errorf("invalid generator type: %s, must be one of: nop, json, winevt, palo-alto, apache-common, apache-combined, apache-error, nginx, postgres, kubernetes, filegen, okta, hostmetrics, traces, wel, fix", g.Type) + return fmt.Errorf("invalid generator type: %s, must be one of: nop, json, winevt, palo-alto, apache-common, apache-combined, apache-error, nginx, postgres, kubernetes, filegen, okta, hostmetrics, traces, wel, fix, f5", g.Type) } return nil diff --git a/internal/config/generator_f5.go b/internal/config/generator_f5.go new file mode 100644 index 00000000..5bc3d2cc --- /dev/null +++ b/internal/config/generator_f5.go @@ -0,0 +1,53 @@ +package config + +import ( + "fmt" + "time" + + "github.com/observiq/blitz/generator/f5/catalog" + + // Register the F5 products so product-name validation can see them. + _ "github.com/observiq/blitz/generator/f5" +) + +// F5GeneratorConfig contains configuration for the multi-product F5 log +// generator. +type F5GeneratorConfig struct { + // Workers is the number of worker goroutines. + Workers int `yaml:"workers,omitempty" mapstructure:"workers,omitempty"` + // Rate is the per-worker emission interval. + Rate time.Duration `yaml:"rate,omitempty" mapstructure:"rate,omitempty"` + // Hostname is the simulated F5 device hostname in the syslog header. + Hostname string `yaml:"hostname,omitempty" mapstructure:"hostname,omitempty"` + // EnabledProducts restricts emission to a subset of product names. + // Empty = all. Valid tokens: ltm, asm, afm, apm, dns, audit, + // nginx-plus, nginx-app-protect, irules, f5os. + EnabledProducts []string `yaml:"enabledProducts,omitempty" mapstructure:"enabledProducts,omitempty"` + // Weights sets the relative mix ratio per product name. Absent = + // weight 1. Empty map = all equal. + Weights map[string]float64 `yaml:"weights,omitempty" mapstructure:"weights,omitempty"` + // Seed is the deterministic RNG seed. Negative randomizes per worker; + // 0+ produces byte-identical output across runs for the same seed. + Seed int64 `yaml:"seed,omitempty" mapstructure:"seed,omitempty"` +} + +// Validate validates the F5 generator configuration. +func (c *F5GeneratorConfig) Validate() error { + if c.Workers < 1 { + return fmt.Errorf("f5 generator workers must be 1 or greater, got %d", c.Workers) + } + if c.Rate <= 0 { + return fmt.Errorf("f5 generator rate must be positive, got %v", c.Rate) + } + for _, name := range c.EnabledProducts { + if _, ok := catalog.Get(name); !ok { + return fmt.Errorf("f5 generator enabledProducts: unknown product %q", name) + } + } + for name := range c.Weights { + if _, ok := catalog.Get(name); !ok { + return fmt.Errorf("f5 generator weights: unknown product %q", name) + } + } + return nil +} diff --git a/internal/config/generator_f5_test.go b/internal/config/generator_f5_test.go new file mode 100644 index 00000000..116e378c --- /dev/null +++ b/internal/config/generator_f5_test.go @@ -0,0 +1,31 @@ +package config + +import ( + "testing" + "time" +) + +func TestF5GeneratorConfigValidate(t *testing.T) { + tests := []struct { + name string + cfg F5GeneratorConfig + wantErr bool + }{ + {"valid all products", F5GeneratorConfig{Workers: 1, Rate: time.Second}, false}, + {"valid subset", F5GeneratorConfig{Workers: 2, Rate: time.Second, EnabledProducts: []string{"ltm", "asm"}}, false}, + {"workers zero", F5GeneratorConfig{Workers: 0, Rate: time.Second}, true}, + {"rate zero", F5GeneratorConfig{Workers: 1, Rate: 0}, true}, + {"unknown product", F5GeneratorConfig{Workers: 1, Rate: time.Second, EnabledProducts: []string{"nope"}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.cfg.Validate() + if tt.wantErr && err == nil { + t.Fatalf("expected error, got nil") + } + if !tt.wantErr && err != nil { + t.Fatalf("expected no error, got %v", err) + } + }) + } +} diff --git a/internal/config/override.go b/internal/config/override.go index e0594187..b3206247 100644 --- a/internal/config/override.go +++ b/internal/config/override.go @@ -249,7 +249,7 @@ func DefaultOverrides() []*Override { NewOverride("metrics.port", "HTTP port for the metrics endpoint", DefaultMetricsPort), NewOverride("generator.count", "total number of logs to generate (0 = unlimited)", 0), NewOverride("onFinish", "behavior when finite generation completes. One of: exit|idle", "exit"), - NewOverride("generator.type", "generator type. One of: nop|json|winevt|wel|palo-alto|apache-common|apache-combined|apache-error|nginx|postgres|kubernetes|filegen|okta|hostmetrics|traces", GeneratorTypeNop), + NewOverride("generator.type", "generator type. One of: nop|json|winevt|wel|palo-alto|apache-common|apache-combined|apache-error|nginx|postgres|kubernetes|filegen|okta|hostmetrics|traces|fix|f5", GeneratorTypeNop), NewOverride("generator.json.workers", "number of JSON generator workers", 1), NewOverride("generator.json.rate", "rate at which logs are generated per worker", 1*time.Second), NewOverride("generator.json.type", "type of log to generate. One of: default|pii", logtypes.LogTypeDefault), diff --git a/internal/dispatch/embed.go b/internal/dispatch/embed.go index d6844f5b..6522e850 100644 --- a/internal/dispatch/embed.go +++ b/internal/dispatch/embed.go @@ -13,6 +13,7 @@ import ( apachegen "github.com/observiq/blitz/generator/apache" apachecombinedgen "github.com/observiq/blitz/generator/apache_combined" apacheerrorgen "github.com/observiq/blitz/generator/apache_error" + f5gen "github.com/observiq/blitz/generator/f5" "github.com/observiq/blitz/generator/filegen" fixgen "github.com/observiq/blitz/generator/fix" "github.com/observiq/blitz/generator/fix/catalog" @@ -172,6 +173,19 @@ func ForEmbed(logger *zap.Logger, genCfg config.Generator, consumers EmbedConsum } mod, err := newFIX(logger, genCfg.FIX, consumers.LogConsumer, tel) return applyHostIdentity(mod, err, env, genCfg.Type) + case config.GeneratorTypeF5: + if err := consumers.requireLog(genCfg.Type); err != nil { + return nil, err + } + mod, err := f5gen.New(logger, f5gen.Config{ + Workers: genCfg.F5.Workers, + Rate: genCfg.F5.Rate, + Hostname: genCfg.F5.Hostname, + EnabledProducts: genCfg.F5.EnabledProducts, + Weights: genCfg.F5.Weights, + Seed: genCfg.F5.Seed, + }, consumers.LogConsumer, tel) + return applyHostIdentity(mod, err, env, genCfg.Type) case config.GeneratorTypeHostMetrics: if err := consumers.requireMetric(genCfg.Type); err != nil { return nil, err