fix(email): crash reporting, context limit, schema validation, cachin…

.gitignore

@@ -36,10 +36,10 @@ frontend/playwright/.cache/
 # Test snapshots
 **/__snapshots__/
 **/*.snap
+**/*.snap.new
 **/test-snapshots/
 **/testsnapshots/
 **/testSnapshots/
-**/*.snap.new
 
 # Coverage and cache output
 coverage/
@@ -58,9 +58,26 @@ contracts/**/*.d
 # Cargo lock (regenerated per environment)
 Cargo.lock
 
+# Criterion benchmark output
+**/criterion/
+services/api/benches/.benchmarks/
 
 # Performance baselines (stored per branch)
 .performance-baselines/
+performance/.performance-baselines/
 
 # Gas benchmark baselines (stored per branch)
 contracts/predict-iq/.gas-benchmarks/
+
+# Temporary and editor files
+*.tmp
+*.bak
+*~
+.vscode/*.log
+
+# Shell scripts generated during debugging
+push_and_create_pr*.sh
+verify-e2e-setup.sh
+test_shutdown.sh
+test_shutdown.ps1
+test_rate_limit.sh

performance/config/alerts.yaml

@@ -259,3 +259,28 @@ groups:
           summary: "Performance degradation alert threshold reached"
           description: "Response time has degraded by more than 5% compared to 24h ago (alert threshold)"
           runbook_url: "https://docs.predictiq.com/runbooks/significant-performance-degradation"
+
+  - name: email_worker
+    interval: 30s
+    rules:
+      - alert: EmailWorkerDown
+        expr: absent(up{job="predictiq-api"}) or increase(worker_crash_total{worker="email_queue_worker"}[60s]) > 0
+        for: 1m
+        labels:
+          severity: critical
+          component: email
+        annotations:
+          summary: "Email queue worker has crashed"
+          description: "The email queue worker has crashed {{ $value }} time(s) in the last 60 seconds. Emails may not be delivered."
+          runbook_url: "https://docs.predictiq.com/runbooks/email-worker-down"
+
+      - alert: EmailWorkerCrashLooping
+        expr: increase(worker_crash_total{worker="email_queue_worker"}[5m]) >= 5
+        for: 0m
+        labels:
+          severity: critical
+          component: email
+        annotations:
+          summary: "Email queue worker is crash-looping"
+          description: "The email queue worker has crashed {{ $value }} times in the last 5 minutes and has been permanently stopped. Manual intervention required."
+          runbook_url: "https://docs.predictiq.com/runbooks/email-worker-down"

services/api/Cargo.toml

@@ -61,4 +61,8 @@ harness = false
 name = "invalidation_tags"
 harness = false
 
+[[bench]]
+name = "email_templates"
+harness = false
+
 [workspace]

services/api/benches/email_templates.rs

@@ -0,0 +1,88 @@
+/// Benchmark: cached vs per-request template compilation.
+///
+/// EmailTemplateEngine compiles all templates once at construction time
+/// (EmailTemplateEngine::new()). This benchmark measures the render cost at
+/// 1 000 iterations to confirm that per-request re-parsing is not occurring.
+///
+/// Template-update policy: templates are embedded via include_str! and compiled
+/// at startup. To pick up a template change, restart the service. Hot-reload is
+/// not supported in production because (a) it would require dynamic file I/O
+/// and a filesystem watcher, and (b) template changes require review and
+/// deployment anyway.
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use predictiq_api::email::templates::EmailTemplateEngine;
+
+fn bench_cached_render(c: &mut Criterion) {
+    // Build once — this is the production path (cached registry).
+    let engine = EmailTemplateEngine::new().expect("engine init");
+    let data = serde_json::json!({
+        "confirm_url": "https://example.com/confirm?token=bench",
+        "email": "bench@example.com"
+    });
+
+    let mut group = c.benchmark_group("email_template_render");
+
+    group.bench_function("cached_1000_renders", |b| {
+        b.iter(|| {
+            for _ in 0..1_000 {
+                let _ = engine.render(
+                    black_box("newsletter_confirmation"),
+                    black_box(&data),
+                );
+            }
+        })
+    });
+
+    // Baseline: cost of constructing a fresh engine per render (per-request).
+    // This is the anti-pattern the issue was filed against; the number should
+    // be dramatically higher than the cached path above.
+    group.bench_function("per_request_engine_1000_renders", |b| {
+        b.iter(|| {
+            for _ in 0..1_000 {
+                let fresh = EmailTemplateEngine::new().expect("engine init");
+                let _ = fresh.render(
+                    black_box("newsletter_confirmation"),
+                    black_box(&data),
+                );
+            }
+        })
+    });
+
+    group.finish();
+}
+
+fn bench_all_templates_cached(c: &mut Criterion) {
+    let engine = EmailTemplateEngine::new().expect("engine init");
+
+    let fixtures: &[(&str, serde_json::Value)] = &[
+        ("newsletter_confirmation", serde_json::json!({
+            "confirm_url": "https://example.com/confirm?token=bench",
+            "email": "bench@example.com"
+        })),
+        ("waitlist_confirmation", serde_json::json!({
+            "email": "bench@example.com"
+        })),
+        ("contact_form_auto_response", serde_json::json!({
+            "name": "Bench User",
+            "subject": "Bench Subject",
+            "message": "Bench message."
+        })),
+        ("welcome_email", serde_json::json!({
+            "name": "Bench User",
+            "dashboard_url": "https://example.com/dashboard",
+            "help_url": "https://example.com/help",
+            "unsubscribe_url": "https://example.com/unsubscribe"
+        })),
+    ];
+
+    let mut group = c.benchmark_group("email_template_all_cached");
+    for (name, data) in fixtures {
+        group.bench_with_input(BenchmarkId::new("render", name), data, |b, d| {
+            b.iter(|| engine.render(black_box(name), black_box(d)))
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(benches, bench_cached_render, bench_all_templates_cached);
+criterion_main!(benches);

services/api/src/email/templates.rs

@@ -76,6 +76,19 @@ impl EmailTemplateEngine {
     }
 
     pub fn render(&self, template_name: &str, data: &Value) -> Result<String> {
+        // Guard: reject oversized context data before allocating in the renderer.
+        // A malicious or buggy caller could pass a multi-MB JSON object; serialising
+        // it inside Handlebars would cause excessive memory allocation.
+        const MAX_CONTEXT_BYTES: usize = 64 * 1024; // 64 KB
+        let serialized_len = data.to_string().len();
+        if serialized_len > MAX_CONTEXT_BYTES {
+            anyhow::bail!(
+                "template context for '{}' exceeds the 64 KB limit ({} bytes)",
+                template_name,
+                serialized_len
+            );
+        }
+
         self.handlebars
             .render(template_name, data)
             .with_context(|| format!("Failed to render template: {}", template_name))
@@ -167,4 +180,185 @@ mod tests {
         let subject = engine.get_subject("newsletter_confirmation", &data);
         assert_eq!(subject, "Confirm your newsletter subscription");
     }
+
+    // ── Context size limit tests (Issue 3) ────────────────────────────────────
+
+    #[test]
+    fn oversized_context_is_rejected() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        // Build a context that exceeds 64 KB when serialised.
+        let big_string = "x".repeat(70_000);
+        let data = json!({
+            "confirm_url": "https://example.com/confirm",
+            "email": big_string
+        });
+        let err = engine.render("newsletter_confirmation", &data).unwrap_err();
+        assert!(
+            err.to_string().contains("64 KB limit"),
+            "error should mention the 64 KB limit, got: {err}"
+        );
+    }
+
+    #[test]
+    fn context_at_limit_boundary_is_accepted() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        // Build a context just under 64 KB.
+        let padding = "a".repeat(60_000);
+        let data = json!({
+            "confirm_url": "https://example.com/confirm?token=abc",
+            "email": padding
+        });
+        // Should not return a size error (may fail rendering due to the raw value
+        // appearing in the template, but we just want the size check to pass).
+        let result = engine.render("newsletter_confirmation", &data);
+        // Size check passes — it may succeed or fail for other reasons, but NOT
+        // because of the 64 KB limit.
+        if let Err(ref e) = result {
+            assert!(
+                !e.to_string().contains("64 KB limit"),
+                "should not hit size limit at {}, err: {e}",
+                serde_json::to_string(&data).unwrap().len()
+            );
+        }
+    }
+
+    // ── Boundary-value tests per template (Issue 4) ───────────────────────────
+
+    // newsletter_confirmation
+
+    #[test]
+    fn newsletter_confirmation_empty_strings() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({ "confirm_url": "", "email": "" });
+        // Strict mode is on; empty strings are valid values — should render.
+        assert!(engine.render("newsletter_confirmation", &data).is_ok());
+    }
+
+    #[test]
+    fn newsletter_confirmation_special_chars() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({
+            "confirm_url": "https://example.com/confirm?token=<script>alert(1)</script>",
+            "email": "user+tag@example.com"
+        });
+        let html = engine.render("newsletter_confirmation", &data).unwrap();
+        // Handlebars HTML-escapes by default; angle brackets must be escaped.
+        assert!(!html.contains("<script>"), "XSS payload must be escaped");
+    }
+
+    #[test]
+    fn newsletter_confirmation_long_strings() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let long_url = format!("https://example.com/confirm?token={}", "a".repeat(2000));
+        let data = json!({ "confirm_url": long_url, "email": "user@example.com" });
+        assert!(engine.render("newsletter_confirmation", &data).is_ok());
+    }
+
+    // waitlist_confirmation
+
+    #[test]
+    fn waitlist_confirmation_empty_email() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({ "email": "" });
+        assert!(engine.render("waitlist_confirmation", &data).is_ok());
+    }
+
+    #[test]
+    fn waitlist_confirmation_special_chars_in_email() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({ "email": "user+test&special=<chars>@example.com" });
+        let html = engine.render("waitlist_confirmation", &data).unwrap();
+        assert!(!html.contains("<chars>"), "angle brackets must be HTML-escaped");
+    }
+
+    #[test]
+    fn waitlist_confirmation_long_email() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let local = "a".repeat(64);
+        let data = json!({ "email": format!("{local}@example.com") });
+        assert!(engine.render("waitlist_confirmation", &data).is_ok());
+    }
+
+    // contact_form_auto_response
+
+    #[test]
+    fn contact_form_empty_fields() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({ "name": "", "subject": "", "message": "" });
+        assert!(engine.render("contact_form_auto_response", &data).is_ok());
+    }
+
+    #[test]
+    fn contact_form_special_chars() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({
+            "name": "<script>alert('xss')</script>",
+            "subject": "Re: Test & <HTML>",
+            "message": "Hello \"world\" & <world>"
+        });
+        let html = engine.render("contact_form_auto_response", &data).unwrap();
+        assert!(!html.contains("<script>"), "script tags must be escaped");
+        assert!(!html.contains("<HTML>"), "angle brackets must be escaped");
+    }
+
+    #[test]
+    fn contact_form_long_message() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({
+            "name": "Test User",
+            "subject": "Long message test",
+            "message": "a".repeat(5000)
+        });
+        assert!(engine.render("contact_form_auto_response", &data).is_ok());
+    }
+
+    // welcome_email
+
+    #[test]
+    fn welcome_email_empty_strings() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({
+            "name": "",
+            "dashboard_url": "",
+            "help_url": "",
+            "unsubscribe_url": ""
+        });
+        assert!(engine.render("welcome_email", &data).is_ok());
+    }
+
+    #[test]
+    fn welcome_email_special_chars_in_name() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({
+            "name": "O'Brien & <Test>",
+            "dashboard_url": "https://example.com/dashboard",
+            "help_url": "https://example.com/help",
+            "unsubscribe_url": "https://example.com/unsubscribe"
+        });
+        let html = engine.render("welcome_email", &data).unwrap();
+        assert!(!html.contains("<Test>"), "angle brackets must be escaped");
+    }
+
+    #[test]
+    fn welcome_email_long_name() {
+        let engine = EmailTemplateEngine::new().unwrap();
+        let data = json!({
+            "name": "A".repeat(200),
+            "dashboard_url": "https://example.com/dashboard",
+            "help_url": "https://example.com/help",
+            "unsubscribe_url": "https://example.com/unsubscribe"
+        });
+        assert!(engine.render("welcome_email", &data).is_ok());
+    }
+
+    // ── Startup validation sanity check ──────────────────────────────────────
+
+    #[test]
+    fn engine_init_validates_all_templates_at_startup() {
+        // If any template has a syntax error or missing variable, new() will fail.
+        assert!(
+            EmailTemplateEngine::new().is_ok(),
+            "All templates must be valid at startup"
+        );
+    }
 }