feat(waf): CrowdSec WAF and IP banning for DHIS2

.github/workflows/ci.yml

@@ -135,6 +135,12 @@ jobs:
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
 
+      - name: Run Molecule tests (crowdsec)
+        run: molecule test -s crowdsec
+        env:
+          PY_COLORS: '1'
+          ANSIBLE_FORCE_COLOR: '1'
+
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4

.gitignore

@@ -31,3 +31,6 @@ deploy/filter_plugins/__pycache__/
 *.pyc
 __pycache__/
 molecule/*/tests/__pycache__/
+
+# Local-only documentation (handoffs, research notes)
+documentation/

README.md

@@ -324,3 +324,4 @@ At this point you should have DHIS2 up and running.
 - [LXC container management](./docs/Basic-LXC-container-Management.md)
 - [Service management with systemctl](./docs/Systemd-Service-Management.md)
 - [SSH connection](./docs/SSH-Connection.md)
+- [CrowdSec WAF and IP banning](./docs/CrowdSec-WAF.md)

changelogs/fragments/crowdsec-waf.yml

@@ -0,0 +1,16 @@
+---
+minor_changes:
+  - |
+    Add a ``crowdsec`` role: CrowdSec Security Engine with AppSec (WAF) and the
+    OpenResty bouncer on the ``[web]`` host, enabled with ``crowdsec_enabled=true``
+    (requires ``proxy=openresty`` and ``ansible_connection=lxd``). Log-based
+    scenarios and the community blocklist ban IPs; AppSec blocks known-CVE and
+    generic attacks in-band. Includes a managed allowlist, optional captcha
+    remediation, Prometheus metrics scraped from the ``[monitoring]`` host, a
+    systemd memory ceiling, and clean switch-off of the bouncer or the whole
+    engine. See ``docs/CrowdSec-WAF.md``.
+known_issues:
+  - |
+    OpenResty's ``access.log``, ``error.log`` and ``perf.log`` are not rotated.
+    CrowdSec adds the combined ``access.log``; log rotation for OpenResty is a
+    separate change.

deploy/inventory/hosts.template

@@ -116,6 +116,18 @@ proxy=nginx
 # Options: letsencrypt, customssl, selfsigned, default(letsencrypt if fqdn is defined else selfsigned )
 TLS_TYPE=letsencrypt
 
+# CrowdSec WAF - the bouncer requires proxy=openresty. Setting it back to false removes CrowdSec.
+crowdsec_enabled=false
+# Shares attacker IPs and the attacks they triggered with CrowdSec and pulls its community
+# blocklist. false = nothing leaves the server, and no community blocklist.
+# crowdsec_online_api_enabled=true
+# AppSec inspects every request in-band and blocks known-CVE exploits; needs the bouncer.
+# crowdsec_appsec_enabled=true
+# captcha remediation needs a site key and a secret key (put the secret in vault.yml).
+# crowdsec_bouncer_bouncing_on_type=ban
+# IPs/CIDRs never blocked, e.g. a facility's shared carrier-NAT egress IP. JSON list:
+# crowdsec_allowlist=["203.0.113.10", "198.51.100.0/24"]
+
 
 # instances group variables,
 [instances:vars]

deploy/roles/create-instance/tasks/proxy/openresty.yml

@@ -59,12 +59,16 @@
     - not (hostvars[item]['fqdn'] | default('') | string | trim | length > 0)
     - hostvars[item]['instance_state'] is undefined
 
+# crowdsec_*.conf files belong to the crowdsec role: crowdsec_openresty.conf is owned by the
+# crowdsec-openresty-bouncer package (not a conffile, so apt never restores it) and
+# crowdsec_access_log.conf is templated by the role. Both load via the conf.d/*.conf
+# wildcard in nginx.conf.
 - name: Find files to delete in /usr/local/openresty/nginx/conf/conf.d/
   ansible.builtin.find:
     paths: /usr/local/openresty/nginx/conf/conf.d/
     patterns: '*'
     file_type: file
-    excludes: "{{ groups['instances'] | map('extract', hostvars, 'fqdn') | map('default', 'default', true) | map('regex_replace', '^\\s+|\\s+$', '') | unique | map('regex_replace', '^$', 'default') | map('regex_replace', '(.+)', '\\1.conf') | list }}"
+    excludes: "{{ (groups['instances'] | map('extract', hostvars, 'fqdn') | map('default', 'default', true) | map('regex_replace', '^\\s+|\\s+$', '') | unique | map('regex_replace', '^$', 'default') | map('regex_replace', '(.+)', '\\1.conf') | list) + ['crowdsec_*.conf'] }}"
   register: found_files
 
 - name: Clean /usr/local/openresty/nginx/conf/conf.d/ directory

deploy/roles/crowdsec/defaults/main.yml

@@ -0,0 +1,54 @@
+---
+crowdsec_enabled: false
+crowdsec_version: "latest"
+
+# LAPI: only the colocated bouncer on this same host ever queries it, so it binds to loopback.
+crowdsec_lapi_port: 8080
+
+# Prometheus metrics endpoint. cscli metrics reads it locally, so it stays on. The same
+# listener also serves Go's /debug/pprof/*, so it's bound to loopback unless a Prometheus
+# server on the [monitoring] host scrapes it; then it's bound to this host's service IP
+# and ufw only lets the monitoring host(s) in.
+crowdsec_metrics_enabled: true
+crowdsec_metrics_port: 6060
+crowdsec_metrics_expose: "{{ (server_monitoring | default('') | trim) in ['grafana', 'prometheus', 'grafana/prometheus'] and groups['monitoring'] | default([]) | length > 0 }}"
+
+# Community threat-intel sharing (CrowdSec Central API): sends attacker IPs and the
+# scenarios they triggered to CrowdSec, and pulls the community blocklist.
+crowdsec_online_api_enabled: true
+
+crowdsec_bouncer_enabled: true
+# stream polls the LAPI every 10 s and caches decisions, so bans hold while the engine
+# restarts. live checks the LAPI per request and lets everything through while it's down.
+crowdsec_bouncer_mode: "stream"               # stream or live
+# captcha (and all) need the captcha keys below. Without them the bouncer serves every
+# captcha decision as a ban. DHIS2 API and Android clients can't solve a captcha anyway.
+crowdsec_bouncer_bouncing_on_type: "ban"      # ban, captcha, or all
+crowdsec_bouncer_fallback_remediation: "ban"  # used for remediation types the bouncer doesn't understand
+crowdsec_bouncer_ret_code: ""                 # empty = bouncer's own default (403); set to override
+crowdsec_bouncer_captcha_provider: "recaptcha"  # recaptcha, hcaptcha, or turnstile
+crowdsec_bouncer_captcha_site_key: ""
+crowdsec_bouncer_captcha_secret_key: ""         # keep in vault.yml
+
+# AppSec (WAF): the bouncer forwards every request to CrowdSec's AppSec listener, which
+# blocks it in-band on a virtual-patching (known CVE) or generic rule match. Needs the bouncer.
+crowdsec_appsec_enabled: true
+crowdsec_appsec_port: 7422
+# passthrough lets requests through while AppSec is unreachable; deny blocks them.
+crowdsec_appsec_failure_action: "passthrough"
+# CrowdSec blocks any request body over its limit by default. DHIS2 takes bodies up to
+# 100M (data and metadata imports), so bodies over the limit are let through instead.
+# Form bodies are still inspected up to the limit; JSON bodies over the limit skip body
+# rules entirely (URI, query-string and header rules still apply). Raising the limit
+# costs engine memory and past ~50 MiB the bouncer times out and passes the request through.
+crowdsec_appsec_max_body_size: 10485760       # bytes
+crowdsec_appsec_body_size_exceeded_action: "partial"  # partial, allow, or drop
+
+# systemd MemoryMax for the engine (empty = no limit). Idle it uses ~150 MiB; AppSec
+# buffers up to crowdsec_appsec_max_body_size per in-flight request. systemd restarts
+# the engine if it is killed at the limit; stream-mode bans hold in the bouncer meanwhile.
+crowdsec_memory_max: "1G"
+
+# IPs/CIDRs CrowdSec never blocks: no WAF inspection, no local or community-blocklist
+# bans. Use it for trusted networks, e.g. a facility's shared carrier-NAT egress IP.
+crowdsec_allowlist: []

deploy/roles/crowdsec/handlers/main.yml

@@ -0,0 +1,16 @@
+---
+# Handlers run in the order listed: OpenResty reloads first, so access.log exists by the
+# time the engine restarts and opens it.
+# Only on OpenResty hosts: reloading a stopped unit starts it, which would clash with a
+# proxy that has replaced OpenResty.
+- name: Crowdsec | Reload OpenResty
+  ansible.builtin.service:
+    name: openresty
+    state: reloaded
+  when: proxy | default('') == 'openresty'
+
+- name: Restart crowdsec
+  ansible.builtin.systemd:
+    name: crowdsec
+    state: restarted
+    daemon_reload: true

deploy/roles/crowdsec/meta/argument_specs.yml

@@ -0,0 +1,179 @@
+---
+argument_specs:
+  main:
+    short_description: "CrowdSec WAF: Security Engine, AppSec and bouncer, colocated with the [web] host's proxy"
+    description:
+      - Installs the CrowdSec Security Engine and the OpenResty bouncer on the same host
+        as the active reverse proxy — no dedicated container/inventory group. LXD-only;
+        ansible_connection=ssh is not supported.
+      - Two layers of protection. AppSec (the WAF) inspects each request in-band and
+        blocks known-CVE exploits (virtual patching) and a few generic attacks before they
+        reach DHIS2. Log-based scenarios ban IPs whose traffic in access.log matches an
+        attack pattern (scans, brute force, probing), and the Central API adds the
+        community blocklist.
+      - The bouncer requires proxy=openresty. The nginx.org package the proxy role installs
+        can't load the lua module crowdsec-nginx-bouncer depends on, and apache2 has no
+        bouncer support. With any other proxy, set crowdsec_bouncer_enabled=false to run the
+        Security Engine alone (detection only). AppSec needs the bouncer.
+      - Collections (crowdsecurity/nginx, crowdsecurity/linux, ...) are installed
+        automatically by the crowdsec package's own postinst (cscli setup unattended),
+        which auto-detects the running proxy. This requires the proxy role's
+        {{ proxy }}-install.yml to have already run on this host in the same play -
+        see the include_role ordering in roles/proxy/tasks/main.yml.
+      - Switching off. crowdsec_bouncer_enabled=false purges the bouncer and deletes its
+        LAPI registration. crowdsec_enabled=false also removes the engine package, its apt
+        repo and the metrics firewall rule, on hosts this role installed it on;
+        /etc/crowdsec and /var/lib/crowdsec are kept.
+    options:
+      crowdsec_enabled:
+        description: >-
+          Master switch. Setting it back to false on a host this role set up removes
+          CrowdSec from it.
+        type: bool
+        default: false
+      crowdsec_version:
+        description: >-
+          crowdsec package version to pin (e.g. "1.8.0"), or "latest" to always take the
+          newest package in the CrowdSec apt repo. A pinned version is installed (up or
+          down) and held with dpkg so `apt upgrade` leaves it alone. Only pins the crowdsec
+          engine package - bouncer packages are versioned independently upstream and are
+          always installed at "latest". Don't pin below 1.7.8: the AppSec body-size hooks
+          arrived in that release (with the 10 MiB limit they configure), and
+          crowdsec_allowlist relies on allowlists clearing matching bans (1.6.9).
+        type: str
+        default: "latest"
+      crowdsec_lapi_port:
+        description: >-
+          Port the local CrowdSec API (LAPI) listens on, on 127.0.0.1. The engine's agent
+          credentials and the bouncer's API_URL are kept pointing at it.
+        type: int
+        default: 8080
+      crowdsec_metrics_enabled:
+        description: >-
+          Enable the Prometheus metrics endpoint. `cscli metrics` reads it, so leave it on
+          unless you have a reason not to.
+        type: bool
+        default: true
+      crowdsec_metrics_port:
+        description: "Port the Prometheus metrics endpoint listens on"
+        type: int
+        default: 6060
+      crowdsec_metrics_expose:
+        description: >-
+          Bind the metrics endpoint to this host's service IP (service_ip, else
+          ansible_host) and let the [monitoring] host(s) in through ufw, for Prometheus to
+          scrape. When false it binds to 127.0.0.1 and the ufw rule is deleted. The same
+          listener serves Go's /debug/pprof/* profiling endpoints, which CrowdSec can't turn
+          off, so exposing it also exposes those to the monitoring host. Defaults to true
+          when server_monitoring uses Prometheus and a [monitoring] host exists.
+        type: bool
+      crowdsec_online_api_enabled:
+        description: >-
+          Opt into CrowdSec's Central API: share attack signals (attacker IPs and the
+          scenarios they triggered) with CrowdSec and pull the community blocklist. Setting it
+          to false on an installed host drops the Central API credentials; setting it back
+          registers a new machine.
+        type: bool
+        default: true
+      crowdsec_bouncer_enabled:
+        description: >-
+          Install and configure the remediation component (bouncer). Requires
+          proxy=openresty. Setting it to false on an installed host removes the bouncer.
+        type: bool
+        default: true
+      crowdsec_bouncer_mode:
+        description: >-
+          stream polls the LAPI on an interval (UPDATE_FREQUENCY, 10 s) and caches decisions,
+          so bans stay enforced while the engine restarts; new bans and unbans take up to one
+          interval to apply. live checks the LAPI per request and lets every request through
+          while the engine is down.
+        type: str
+        default: "stream"
+        choices: ["live", "stream"]
+      crowdsec_bouncer_bouncing_on_type:
+        description: >-
+          Which remediation types from the LAPI the bouncer enforces. captcha and all need
+          crowdsec_bouncer_captcha_site_key and crowdsec_bouncer_captcha_secret_key; with ban,
+          captcha decisions are ignored.
+        type: str
+        default: "ban"
+        choices: ["all", "ban", "captcha"]
+      crowdsec_bouncer_fallback_remediation:
+        description: >-
+          Remediation applied when the LAPI returns a type the bouncer doesn't recognize,
+          and when AppSec is unreachable with crowdsec_appsec_failure_action=deny. captcha
+          needs the captcha keys.
+        type: str
+        default: "ban"
+        choices: ["ban", "captcha"]
+      crowdsec_bouncer_ret_code:
+        description: >-
+          HTTP status code returned for banned requests. Empty keeps the bouncer's own
+          default (403). The bouncer accepts only the codes listed here; anything else
+          (429, for example) is replaced with 403 and logged to error.log.
+        type: str
+        default: ""
+        choices: ["", "200", "202", "204", "301", "302", "400", "401", "403", "404", "405", "406", "444", "500"]
+      crowdsec_bouncer_captcha_provider:
+        description: "Captcha service the bouncer's challenge page uses"
+        type: str
+        default: "recaptcha"
+        choices: ["recaptcha", "hcaptcha", "turnstile"]
+      crowdsec_bouncer_captcha_site_key:
+        description: "Site key from the captcha provider"
+        type: str
+        default: ""
+      crowdsec_bouncer_captcha_secret_key:
+        description: "Secret key from the captcha provider. Keep it in vault.yml."
+        type: str
+        default: ""
+        no_log: true
+      crowdsec_appsec_enabled:
+        description: >-
+          Run the AppSec component (WAF). The bouncer sends every request to it, and it
+          blocks in-band on a crowdsecurity/appsec-virtual-patching or
+          crowdsecurity/appsec-generic-rules match. Needs crowdsec_bouncer_enabled.
+        type: bool
+        default: true
+      crowdsec_appsec_port:
+        description: "Port the AppSec listener binds to, on 127.0.0.1"
+        type: int
+        default: 7422
+      crowdsec_appsec_failure_action:
+        description: >-
+          What the bouncer does when AppSec doesn't answer: passthrough lets the request
+          through, deny applies crowdsec_bouncer_fallback_remediation.
+        type: str
+        default: "passthrough"
+        choices: ["passthrough", "deny"]
+      crowdsec_appsec_max_body_size:
+        description: >-
+          Largest request body, in bytes, AppSec inspects in full. See
+          crowdsec_appsec_body_size_exceeded_action for larger ones.
+        type: int
+        default: 10485760
+      crowdsec_appsec_body_size_exceeded_action:
+        description: >-
+          What AppSec does with a body over crowdsec_appsec_max_body_size. partial lets
+          the request through and inspects form bodies up to the limit; JSON bodies over
+          the limit skip body rules (URI, query-string and header rules still apply).
+          allow skips body inspection; drop blocks the request and raises an alert against
+          the sender's IP, which blocks DHIS2 imports larger than the limit.
+        type: str
+        default: "partial"
+        choices: ["partial", "allow", "drop"]
+      crowdsec_memory_max:
+        description: >-
+          systemd MemoryMax for the crowdsec service, e.g. 1G or 768M; empty removes the
+          limit. The engine idles around 150 MiB and buffers up to
+          crowdsec_appsec_max_body_size per in-flight AppSec request.
+        type: str
+        default: "1G"
+      crowdsec_allowlist:
+        description: >-
+          IPs and CIDRs CrowdSec never acts on: no AppSec inspection, no local or
+          community-blocklist decisions. Adding an entry also lifts existing bans on it;
+          AppSec picks up changes within about a minute.
+        type: list
+        elements: str
+        default: []

deploy/roles/crowdsec/meta/main.yml

@@ -0,0 +1,19 @@
+---
+galaxy_info:
+  author: DHIS2
+  description: CrowdSec Security Engine + bouncer, colocated on the [web] host
+  license: BSD-3-Clause
+  min_ansible_version: "2.14"
+  platforms:
+    - name: Ubuntu
+      versions:
+        - jammy    # 22.04
+        - noble    # 24.04
+        # Bouncer is OpenResty-only on both - see the pre-flight assert in tasks/main.yml.
+  galaxy_tags:
+    - crowdsec
+    - waf
+    - security
+    - dhis2
+
+dependencies: []