diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa6aa86..bc005db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,6 +135,12 @@ jobs: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' + - name: Run Molecule tests (crowdsec) + run: molecule test -s crowdsec + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + - name: Upload test results if: always() uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index e559ae5..f3fa294 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ deploy/filter_plugins/__pycache__/ *.pyc __pycache__/ molecule/*/tests/__pycache__/ + +# Local-only documentation (handoffs, research notes) +documentation/ diff --git a/README.md b/README.md index 3abb06a..ba1fd59 100644 --- a/README.md +++ b/README.md @@ -324,3 +324,4 @@ At this point you should have DHIS2 up and running. - [LXC container management](./docs/Basic-LXC-container-Management.md) - [Service management with systemctl](./docs/Systemd-Service-Management.md) - [SSH connection](./docs/SSH-Connection.md) +- [CrowdSec WAF and IP banning](./docs/CrowdSec-WAF.md) diff --git a/changelogs/fragments/crowdsec-waf.yml b/changelogs/fragments/crowdsec-waf.yml new file mode 100644 index 0000000..f282182 --- /dev/null +++ b/changelogs/fragments/crowdsec-waf.yml @@ -0,0 +1,16 @@ +--- +minor_changes: + - | + Add a ``crowdsec`` role: CrowdSec Security Engine with AppSec (WAF) and the + OpenResty bouncer on the ``[web]`` host, enabled with ``crowdsec_enabled=true`` + (requires ``proxy=openresty`` and ``ansible_connection=lxd``). Log-based + scenarios and the community blocklist ban IPs; AppSec blocks known-CVE and + generic attacks in-band. Includes a managed allowlist, optional captcha + remediation, Prometheus metrics scraped from the ``[monitoring]`` host, a + systemd memory ceiling, and clean switch-off of the bouncer or the whole + engine. See ``docs/CrowdSec-WAF.md``. +known_issues: + - | + OpenResty's ``access.log``, ``error.log`` and ``perf.log`` are not rotated. + CrowdSec adds the combined ``access.log``; log rotation for OpenResty is a + separate change. diff --git a/deploy/inventory/hosts.template b/deploy/inventory/hosts.template index d10be31..d27e577 100644 --- a/deploy/inventory/hosts.template +++ b/deploy/inventory/hosts.template @@ -116,6 +116,18 @@ proxy=nginx # Options: letsencrypt, customssl, selfsigned, default(letsencrypt if fqdn is defined else selfsigned ) TLS_TYPE=letsencrypt +# CrowdSec WAF - the bouncer requires proxy=openresty. Setting it back to false removes CrowdSec. +crowdsec_enabled=false +# Shares attacker IPs and the attacks they triggered with CrowdSec and pulls its community +# blocklist. false = nothing leaves the server, and no community blocklist. +# crowdsec_online_api_enabled=true +# AppSec inspects every request in-band and blocks known-CVE exploits; needs the bouncer. +# crowdsec_appsec_enabled=true +# captcha remediation needs a site key and a secret key (put the secret in vault.yml). +# crowdsec_bouncer_bouncing_on_type=ban +# IPs/CIDRs never blocked, e.g. a facility's shared carrier-NAT egress IP. JSON list: +# crowdsec_allowlist=["203.0.113.10", "198.51.100.0/24"] + # instances group variables, [instances:vars] diff --git a/deploy/roles/create-instance/tasks/proxy/openresty.yml b/deploy/roles/create-instance/tasks/proxy/openresty.yml index 61ba4c2..3031473 100644 --- a/deploy/roles/create-instance/tasks/proxy/openresty.yml +++ b/deploy/roles/create-instance/tasks/proxy/openresty.yml @@ -59,12 +59,16 @@ - not (hostvars[item]['fqdn'] | default('') | string | trim | length > 0) - hostvars[item]['instance_state'] is undefined +# crowdsec_*.conf files belong to the crowdsec role: crowdsec_openresty.conf is owned by the +# crowdsec-openresty-bouncer package (not a conffile, so apt never restores it) and +# crowdsec_access_log.conf is templated by the role. Both load via the conf.d/*.conf +# wildcard in nginx.conf. - name: Find files to delete in /usr/local/openresty/nginx/conf/conf.d/ ansible.builtin.find: paths: /usr/local/openresty/nginx/conf/conf.d/ patterns: '*' file_type: file - excludes: "{{ groups['instances'] | map('extract', hostvars, 'fqdn') | map('default', 'default', true) | map('regex_replace', '^\\s+|\\s+$', '') | unique | map('regex_replace', '^$', 'default') | map('regex_replace', '(.+)', '\\1.conf') | list }}" + excludes: "{{ (groups['instances'] | map('extract', hostvars, 'fqdn') | map('default', 'default', true) | map('regex_replace', '^\\s+|\\s+$', '') | unique | map('regex_replace', '^$', 'default') | map('regex_replace', '(.+)', '\\1.conf') | list) + ['crowdsec_*.conf'] }}" register: found_files - name: Clean /usr/local/openresty/nginx/conf/conf.d/ directory diff --git a/deploy/roles/crowdsec/defaults/main.yml b/deploy/roles/crowdsec/defaults/main.yml new file mode 100644 index 0000000..75624c0 --- /dev/null +++ b/deploy/roles/crowdsec/defaults/main.yml @@ -0,0 +1,54 @@ +--- +crowdsec_enabled: false +crowdsec_version: "latest" + +# LAPI: only the colocated bouncer on this same host ever queries it, so it binds to loopback. +crowdsec_lapi_port: 8080 + +# Prometheus metrics endpoint. cscli metrics reads it locally, so it stays on. The same +# listener also serves Go's /debug/pprof/*, so it's bound to loopback unless a Prometheus +# server on the [monitoring] host scrapes it; then it's bound to this host's service IP +# and ufw only lets the monitoring host(s) in. +crowdsec_metrics_enabled: true +crowdsec_metrics_port: 6060 +crowdsec_metrics_expose: "{{ (server_monitoring | default('') | trim) in ['grafana', 'prometheus', 'grafana/prometheus'] and groups['monitoring'] | default([]) | length > 0 }}" + +# Community threat-intel sharing (CrowdSec Central API): sends attacker IPs and the +# scenarios they triggered to CrowdSec, and pulls the community blocklist. +crowdsec_online_api_enabled: true + +crowdsec_bouncer_enabled: true +# stream polls the LAPI every 10 s and caches decisions, so bans hold while the engine +# restarts. live checks the LAPI per request and lets everything through while it's down. +crowdsec_bouncer_mode: "stream" # stream or live +# captcha (and all) need the captcha keys below. Without them the bouncer serves every +# captcha decision as a ban. DHIS2 API and Android clients can't solve a captcha anyway. +crowdsec_bouncer_bouncing_on_type: "ban" # ban, captcha, or all +crowdsec_bouncer_fallback_remediation: "ban" # used for remediation types the bouncer doesn't understand +crowdsec_bouncer_ret_code: "" # empty = bouncer's own default (403); set to override +crowdsec_bouncer_captcha_provider: "recaptcha" # recaptcha, hcaptcha, or turnstile +crowdsec_bouncer_captcha_site_key: "" +crowdsec_bouncer_captcha_secret_key: "" # keep in vault.yml + +# AppSec (WAF): the bouncer forwards every request to CrowdSec's AppSec listener, which +# blocks it in-band on a virtual-patching (known CVE) or generic rule match. Needs the bouncer. +crowdsec_appsec_enabled: true +crowdsec_appsec_port: 7422 +# passthrough lets requests through while AppSec is unreachable; deny blocks them. +crowdsec_appsec_failure_action: "passthrough" +# CrowdSec blocks any request body over its limit by default. DHIS2 takes bodies up to +# 100M (data and metadata imports), so bodies over the limit are let through instead. +# Form bodies are still inspected up to the limit; JSON bodies over the limit skip body +# rules entirely (URI, query-string and header rules still apply). Raising the limit +# costs engine memory and past ~50 MiB the bouncer times out and passes the request through. +crowdsec_appsec_max_body_size: 10485760 # bytes +crowdsec_appsec_body_size_exceeded_action: "partial" # partial, allow, or drop + +# systemd MemoryMax for the engine (empty = no limit). Idle it uses ~150 MiB; AppSec +# buffers up to crowdsec_appsec_max_body_size per in-flight request. systemd restarts +# the engine if it is killed at the limit; stream-mode bans hold in the bouncer meanwhile. +crowdsec_memory_max: "1G" + +# IPs/CIDRs CrowdSec never blocks: no WAF inspection, no local or community-blocklist +# bans. Use it for trusted networks, e.g. a facility's shared carrier-NAT egress IP. +crowdsec_allowlist: [] diff --git a/deploy/roles/crowdsec/handlers/main.yml b/deploy/roles/crowdsec/handlers/main.yml new file mode 100644 index 0000000..dae16fa --- /dev/null +++ b/deploy/roles/crowdsec/handlers/main.yml @@ -0,0 +1,16 @@ +--- +# Handlers run in the order listed: OpenResty reloads first, so access.log exists by the +# time the engine restarts and opens it. +# Only on OpenResty hosts: reloading a stopped unit starts it, which would clash with a +# proxy that has replaced OpenResty. +- name: Crowdsec | Reload OpenResty + ansible.builtin.service: + name: openresty + state: reloaded + when: proxy | default('') == 'openresty' + +- name: Restart crowdsec + ansible.builtin.systemd: + name: crowdsec + state: restarted + daemon_reload: true diff --git a/deploy/roles/crowdsec/meta/argument_specs.yml b/deploy/roles/crowdsec/meta/argument_specs.yml new file mode 100644 index 0000000..e67158b --- /dev/null +++ b/deploy/roles/crowdsec/meta/argument_specs.yml @@ -0,0 +1,179 @@ +--- +argument_specs: + main: + short_description: "CrowdSec WAF: Security Engine, AppSec and bouncer, colocated with the [web] host's proxy" + description: + - Installs the CrowdSec Security Engine and the OpenResty bouncer on the same host + as the active reverse proxy — no dedicated container/inventory group. LXD-only; + ansible_connection=ssh is not supported. + - Two layers of protection. AppSec (the WAF) inspects each request in-band and + blocks known-CVE exploits (virtual patching) and a few generic attacks before they + reach DHIS2. Log-based scenarios ban IPs whose traffic in access.log matches an + attack pattern (scans, brute force, probing), and the Central API adds the + community blocklist. + - The bouncer requires proxy=openresty. The nginx.org package the proxy role installs + can't load the lua module crowdsec-nginx-bouncer depends on, and apache2 has no + bouncer support. With any other proxy, set crowdsec_bouncer_enabled=false to run the + Security Engine alone (detection only). AppSec needs the bouncer. + - Collections (crowdsecurity/nginx, crowdsecurity/linux, ...) are installed + automatically by the crowdsec package's own postinst (cscli setup unattended), + which auto-detects the running proxy. This requires the proxy role's + {{ proxy }}-install.yml to have already run on this host in the same play - + see the include_role ordering in roles/proxy/tasks/main.yml. + - Switching off. crowdsec_bouncer_enabled=false purges the bouncer and deletes its + LAPI registration. crowdsec_enabled=false also removes the engine package, its apt + repo and the metrics firewall rule, on hosts this role installed it on; + /etc/crowdsec and /var/lib/crowdsec are kept. + options: + crowdsec_enabled: + description: >- + Master switch. Setting it back to false on a host this role set up removes + CrowdSec from it. + type: bool + default: false + crowdsec_version: + description: >- + crowdsec package version to pin (e.g. "1.8.0"), or "latest" to always take the + newest package in the CrowdSec apt repo. A pinned version is installed (up or + down) and held with dpkg so `apt upgrade` leaves it alone. Only pins the crowdsec + engine package - bouncer packages are versioned independently upstream and are + always installed at "latest". Don't pin below 1.7.8: the AppSec body-size hooks + arrived in that release (with the 10 MiB limit they configure), and + crowdsec_allowlist relies on allowlists clearing matching bans (1.6.9). + type: str + default: "latest" + crowdsec_lapi_port: + description: >- + Port the local CrowdSec API (LAPI) listens on, on 127.0.0.1. The engine's agent + credentials and the bouncer's API_URL are kept pointing at it. + type: int + default: 8080 + crowdsec_metrics_enabled: + description: >- + Enable the Prometheus metrics endpoint. `cscli metrics` reads it, so leave it on + unless you have a reason not to. + type: bool + default: true + crowdsec_metrics_port: + description: "Port the Prometheus metrics endpoint listens on" + type: int + default: 6060 + crowdsec_metrics_expose: + description: >- + Bind the metrics endpoint to this host's service IP (service_ip, else + ansible_host) and let the [monitoring] host(s) in through ufw, for Prometheus to + scrape. When false it binds to 127.0.0.1 and the ufw rule is deleted. The same + listener serves Go's /debug/pprof/* profiling endpoints, which CrowdSec can't turn + off, so exposing it also exposes those to the monitoring host. Defaults to true + when server_monitoring uses Prometheus and a [monitoring] host exists. + type: bool + crowdsec_online_api_enabled: + description: >- + Opt into CrowdSec's Central API: share attack signals (attacker IPs and the + scenarios they triggered) with CrowdSec and pull the community blocklist. Setting it + to false on an installed host drops the Central API credentials; setting it back + registers a new machine. + type: bool + default: true + crowdsec_bouncer_enabled: + description: >- + Install and configure the remediation component (bouncer). Requires + proxy=openresty. Setting it to false on an installed host removes the bouncer. + type: bool + default: true + crowdsec_bouncer_mode: + description: >- + stream polls the LAPI on an interval (UPDATE_FREQUENCY, 10 s) and caches decisions, + so bans stay enforced while the engine restarts; new bans and unbans take up to one + interval to apply. live checks the LAPI per request and lets every request through + while the engine is down. + type: str + default: "stream" + choices: ["live", "stream"] + crowdsec_bouncer_bouncing_on_type: + description: >- + Which remediation types from the LAPI the bouncer enforces. captcha and all need + crowdsec_bouncer_captcha_site_key and crowdsec_bouncer_captcha_secret_key; with ban, + captcha decisions are ignored. + type: str + default: "ban" + choices: ["all", "ban", "captcha"] + crowdsec_bouncer_fallback_remediation: + description: >- + Remediation applied when the LAPI returns a type the bouncer doesn't recognize, + and when AppSec is unreachable with crowdsec_appsec_failure_action=deny. captcha + needs the captcha keys. + type: str + default: "ban" + choices: ["ban", "captcha"] + crowdsec_bouncer_ret_code: + description: >- + HTTP status code returned for banned requests. Empty keeps the bouncer's own + default (403). The bouncer accepts only the codes listed here; anything else + (429, for example) is replaced with 403 and logged to error.log. + type: str + default: "" + choices: ["", "200", "202", "204", "301", "302", "400", "401", "403", "404", "405", "406", "444", "500"] + crowdsec_bouncer_captcha_provider: + description: "Captcha service the bouncer's challenge page uses" + type: str + default: "recaptcha" + choices: ["recaptcha", "hcaptcha", "turnstile"] + crowdsec_bouncer_captcha_site_key: + description: "Site key from the captcha provider" + type: str + default: "" + crowdsec_bouncer_captcha_secret_key: + description: "Secret key from the captcha provider. Keep it in vault.yml." + type: str + default: "" + no_log: true + crowdsec_appsec_enabled: + description: >- + Run the AppSec component (WAF). The bouncer sends every request to it, and it + blocks in-band on a crowdsecurity/appsec-virtual-patching or + crowdsecurity/appsec-generic-rules match. Needs crowdsec_bouncer_enabled. + type: bool + default: true + crowdsec_appsec_port: + description: "Port the AppSec listener binds to, on 127.0.0.1" + type: int + default: 7422 + crowdsec_appsec_failure_action: + description: >- + What the bouncer does when AppSec doesn't answer: passthrough lets the request + through, deny applies crowdsec_bouncer_fallback_remediation. + type: str + default: "passthrough" + choices: ["passthrough", "deny"] + crowdsec_appsec_max_body_size: + description: >- + Largest request body, in bytes, AppSec inspects in full. See + crowdsec_appsec_body_size_exceeded_action for larger ones. + type: int + default: 10485760 + crowdsec_appsec_body_size_exceeded_action: + description: >- + What AppSec does with a body over crowdsec_appsec_max_body_size. partial lets + the request through and inspects form bodies up to the limit; JSON bodies over + the limit skip body rules (URI, query-string and header rules still apply). + allow skips body inspection; drop blocks the request and raises an alert against + the sender's IP, which blocks DHIS2 imports larger than the limit. + type: str + default: "partial" + choices: ["partial", "allow", "drop"] + crowdsec_memory_max: + description: >- + systemd MemoryMax for the crowdsec service, e.g. 1G or 768M; empty removes the + limit. The engine idles around 150 MiB and buffers up to + crowdsec_appsec_max_body_size per in-flight AppSec request. + type: str + default: "1G" + crowdsec_allowlist: + description: >- + IPs and CIDRs CrowdSec never acts on: no AppSec inspection, no local or + community-blocklist decisions. Adding an entry also lifts existing bans on it; + AppSec picks up changes within about a minute. + type: list + elements: str + default: [] diff --git a/deploy/roles/crowdsec/meta/main.yml b/deploy/roles/crowdsec/meta/main.yml new file mode 100644 index 0000000..97592b7 --- /dev/null +++ b/deploy/roles/crowdsec/meta/main.yml @@ -0,0 +1,19 @@ +--- +galaxy_info: + author: DHIS2 + description: CrowdSec Security Engine + bouncer, colocated on the [web] host + license: BSD-3-Clause + min_ansible_version: "2.14" + platforms: + - name: Ubuntu + versions: + - jammy # 22.04 + - noble # 24.04 + # Bouncer is OpenResty-only on both - see the pre-flight assert in tasks/main.yml. + galaxy_tags: + - crowdsec + - waf + - security + - dhis2 + +dependencies: [] diff --git a/deploy/roles/crowdsec/tasks/allowlist.yml b/deploy/roles/crowdsec/tasks/allowlist.yml new file mode 100644 index 0000000..7e6e597 --- /dev/null +++ b/deploy/roles/crowdsec/tasks/allowlist.yml @@ -0,0 +1,52 @@ +--- +# A centralized allowlist, not an s02-enrich whitelist parser: the parser only drops +# locally detected alerts, while an allowlist also keeps community-blocklist entries and +# AppSec blocks off the listed IPs, and adding an entry deletes its existing bans. The +# role owns this one allowlist entirely. AppSec re-reads allowlists about once a minute. +- name: Crowdsec | Read the role's allowlist + ansible.builtin.command: "cscli allowlists inspect {{ crowdsec_allowlist_name }} -o json" + register: crowdsec_allowlist_current + changed_when: false + failed_when: + - crowdsec_allowlist_current.rc != 0 + - "'not found' not in crowdsec_allowlist_current.stderr" + +# cscli stores each value exactly as given, so plain string comparison is enough. +- name: Crowdsec | Compare the allowlist with crowdsec_allowlist + ansible.builtin.set_fact: + crowdsec_allowlist_missing: "{{ crowdsec_allowlist | map('string') | list | difference(crowdsec_allowlist_present) }}" + crowdsec_allowlist_extra: "{{ crowdsec_allowlist_present | difference(crowdsec_allowlist | map('string') | list) }}" + vars: + crowdsec_allowlist_present: >- + {{ ((crowdsec_allowlist_current.stdout | from_json)['items'] | default([], true) | map(attribute='value') | list) + if crowdsec_allowlist_current.rc == 0 else [] }} + +- name: Crowdsec | Create the allowlist + ansible.builtin.command: >- + cscli allowlists create {{ crowdsec_allowlist_name }} + -d "Trusted networks, managed by Ansible (crowdsec_allowlist)" + when: + - crowdsec_allowlist | length > 0 + - crowdsec_allowlist_current.rc != 0 + changed_when: true + +- name: Crowdsec | Add new allowlist entries + ansible.builtin.command: + argv: "{{ ['cscli', 'allowlists', 'add', crowdsec_allowlist_name] + crowdsec_allowlist_missing + ['-d', 'crowdsec_allowlist'] }}" + when: crowdsec_allowlist_missing | length > 0 + changed_when: true + +- name: Crowdsec | Remove entries dropped from crowdsec_allowlist + ansible.builtin.command: + argv: "{{ ['cscli', 'allowlists', 'remove', crowdsec_allowlist_name] + crowdsec_allowlist_extra }}" + when: + - crowdsec_allowlist | length > 0 + - crowdsec_allowlist_extra | length > 0 + changed_when: true + +- name: Crowdsec | Delete the allowlist when crowdsec_allowlist is empty + ansible.builtin.command: "cscli allowlists delete {{ crowdsec_allowlist_name }}" + when: + - crowdsec_allowlist | length == 0 + - crowdsec_allowlist_current.rc == 0 + changed_when: true diff --git a/deploy/roles/crowdsec/tasks/appsec.yml b/deploy/roles/crowdsec/tasks/appsec.yml new file mode 100644 index 0000000..226a3e6 --- /dev/null +++ b/deploy/roles/crowdsec/tasks/appsec.yml @@ -0,0 +1,58 @@ +--- +# virtual-patching: in-band rules for known CVEs, each scoped to the vulnerable URL or +# parameter. generic-rules: a few broad ones (FreeMarker SSTI, WordPress uploads). +- name: Crowdsec | Install AppSec rule collections + ansible.builtin.command: + cmd: "cscli collections install {{ item }}" + creates: "/etc/crowdsec/collections/{{ item | basename }}.yaml" + loop: "{{ crowdsec_appsec_collections }}" + register: crowdsec_appsec_collection_install + until: crowdsec_appsec_collection_install is succeeded + retries: 3 + delay: 10 + when: crowdsec_appsec_active | bool + notify: Restart crowdsec + +- name: Crowdsec | Ensure the local AppSec config directory exists + ansible.builtin.file: + path: /etc/crowdsec/appsec-configs + state: directory + owner: root + group: root + mode: "0750" + when: crowdsec_appsec_active | bool + +- name: Crowdsec | Template AppSec body-size limits + ansible.builtin.template: + src: appsec-body-limits.yaml.j2 + dest: /etc/crowdsec/appsec-configs/dhis2-body-limits.yaml + owner: root + group: root + mode: "0640" + when: crowdsec_appsec_active | bool + notify: Restart crowdsec + +- name: Crowdsec | Template AppSec acquisition + ansible.builtin.template: + src: acquis-appsec.yaml.j2 + dest: /etc/crowdsec/acquis.d/appsec.yaml + owner: root + group: root + mode: "0640" + when: crowdsec_appsec_active | bool + notify: Restart crowdsec + +# The collections stay installed: their scenarios only act on AppSec events, and none +# arrive once the listener is gone. +- name: Crowdsec | Remove AppSec listener + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/crowdsec/acquis.d/appsec.yaml + - /etc/crowdsec/appsec-configs/dhis2-body-limits.yaml + when: not (crowdsec_appsec_active | bool) + notify: Restart crowdsec + +- name: Crowdsec | Flush handlers so the AppSec listener is up before the bouncer uses it + ansible.builtin.meta: flush_handlers diff --git a/deploy/roles/crowdsec/tasks/configure-bouncer-openresty.yml b/deploy/roles/crowdsec/tasks/configure-bouncer-openresty.yml new file mode 100644 index 0000000..824167d --- /dev/null +++ b/deploy/roles/crowdsec/tasks/configure-bouncer-openresty.yml @@ -0,0 +1,116 @@ +--- +- name: Crowdsec | Install OpenResty bouncer prerequisites + ansible.builtin.apt: + name: + - openresty-opm + - gettext-base + state: present + update_cache: true + cache_valid_time: 3600 + +# crowdsec-openresty-bouncer's postinst auto-registers a fresh API key with the local LAPI +# (cscli bouncers add) and drops crowdsec_openresty.conf into conf.d/. The proxy role's +# nginx.conf already includes conf.d/*.conf inside http {}, so no explicit include is +# needed; adding one declares lua_package_path twice and fails `openresty -t`. +- name: Crowdsec | Install OpenResty bouncer package + ansible.builtin.apt: + name: crowdsec-openresty-bouncer + state: latest + notify: Crowdsec | Reload OpenResty + +# The package ships the conffile world-readable; it holds the bouncer's LAPI API key. +# Only the OpenResty master (root) reads it, in init_by_lua_block. +- name: Crowdsec | Restrict the bouncer config to root + ansible.builtin.file: + path: "{{ crowdsec_bouncer_conf }}" + owner: root + group: root + mode: "0600" + +- name: Crowdsec | Read OpenResty bouncer config + ansible.builtin.slurp: + src: "{{ crowdsec_bouncer_conf }}" + register: crowdsec_openresty_bouncer_conf + no_log: true + +# A failed registration leaves the package's envsubst placeholder (API_KEY=${API_KEY}) in +# place. OpenResty still starts, and every request is let through. +- name: Crowdsec | Assert OpenResty bouncer auto-registered with a real API key + ansible.builtin.assert: + that: + - "(crowdsec_openresty_bouncer_conf.content | b64decode) is regex('^API_KEY=(?!\\$\\{|<)\\S+$', multiline=True)" + fail_msg: >- + crowdsec-openresty-bouncer failed to auto-register against the local LAPI + (cscli/crowdsec wasn't reachable during package install). Run + `cscli bouncers add crowdsec-openresty-bouncer` manually and set API_KEY in + {{ crowdsec_bouncer_conf }}, then re-run this role. + +# diff: false on the edits below - the file holds the bouncer's API key. +- name: Crowdsec | Configure OpenResty bouncer LAPI URL, AppSec and remediation behavior + ansible.builtin.lineinfile: + path: "{{ crowdsec_bouncer_conf }}" + regexp: "^{{ item.key }}=" + line: "{{ item.key }}={{ item.value }}" + diff: false + loop: + - { key: "API_URL", value: "{{ crowdsec_lapi_url }}" } + - { key: "MODE", value: "{{ crowdsec_bouncer_mode }}" } + - { key: "BOUNCING_ON_TYPE", value: "{{ crowdsec_bouncer_bouncing_on_type }}" } + - { key: "FALLBACK_REMEDIATION", value: "{{ crowdsec_bouncer_fallback_remediation }}" } + # Empty is the package's own default and means 403. + - { key: "RET_CODE", value: "{{ crowdsec_bouncer_ret_code }}" } + # Empty turns AppSec off in the bouncer. + - { key: "APPSEC_URL", value: "{{ crowdsec_appsec_url if crowdsec_appsec_active | bool else '' }}" } + - { key: "APPSEC_FAILURE_ACTION", value: "{{ crowdsec_appsec_failure_action }}" } + - { key: "CAPTCHA_PROVIDER", value: "{{ crowdsec_bouncer_captcha_provider }}" } + - { key: "SITE_KEY", value: "{{ crowdsec_bouncer_captcha_site_key }}" } + loop_control: + label: "{{ item.key }}" + notify: Crowdsec | Reload OpenResty + +- name: Crowdsec | Configure OpenResty bouncer captcha secret key + ansible.builtin.lineinfile: + path: "{{ crowdsec_bouncer_conf }}" + regexp: "^SECRET_KEY=" + line: "SECRET_KEY={{ crowdsec_bouncer_captcha_secret_key }}" + diff: false + no_log: true + notify: Crowdsec | Reload OpenResty + +- name: Crowdsec | Flush handlers so OpenResty runs the final bouncer config + ansible.builtin.meta: flush_handlers + +# A bouncer that never pulls (edited crowdsec_openresty.conf without the +# init_worker_by_lua_block, a stale API key, an unreachable API_URL) lets every request +# through without an error anywhere. Stream mode pulls every 10 s, so a pull newer than +# the start of this check proves the whole path. Live mode pulls per request, so this +# only applies to stream. +- name: Crowdsec | Verify the OpenResty bouncer enforces decisions + when: crowdsec_bouncer_mode == 'stream' + block: + - name: Crowdsec | Note the time before checking the bouncer + ansible.builtin.command: date -u +%Y-%m-%dT%H:%M:%SZ + register: crowdsec_bouncer_check_start + changed_when: false + + - name: Crowdsec | Wait for the OpenResty bouncer to pull decisions from the LAPI + ansible.builtin.command: cscli bouncers list -o json + register: crowdsec_bouncer_pull + changed_when: false + retries: 12 + delay: 5 + until: >- + crowdsec_bouncer_pull.stdout | from_json + | selectattr('name', 'match', '^crowdsec-openresty-bouncer') + | map(attribute='last_pull') | select('string') + | map('regex_replace', '\.\d+Z$', 'Z') + | select('>=', crowdsec_bouncer_check_start.stdout) | list | length > 0 + rescue: + - name: Crowdsec | Fail with the bouncer diagnosis + ansible.builtin.fail: + msg: >- + crowdsec-openresty-bouncer has not pulled decisions from the LAPI since + {{ crowdsec_bouncer_check_start.stdout | default('the check started') }}, so bans + are not enforced. Check that {{ crowdsec_openresty_conf_dir }}/crowdsec_openresty.conf + is the package's own file (dpkg --verify crowdsec-openresty-bouncer), that API_KEY in + {{ crowdsec_bouncer_conf }} matches `cscli bouncers list`, and OpenResty's error.log. diff --git a/deploy/roles/crowdsec/tasks/deploy.yml b/deploy/roles/crowdsec/tasks/deploy.yml new file mode 100644 index 0000000..c10001d --- /dev/null +++ b/deploy/roles/crowdsec/tasks/deploy.yml @@ -0,0 +1,53 @@ +--- +# import_tasks, not include_tasks: the sub-tags must reach the tasks inside so that +# --tags crowdsec-install / crowdsec-configure / crowdsec-remove select them. +- name: Crowdsec | Install and configure Security Engine + ansible.builtin.import_tasks: install-engine.yml + when: crowdsec_enabled | bool + tags: [crowdsec-install] + +# Before the bouncer: its APPSEC_URL should point at a listener that's already up. +- name: Crowdsec | Configure AppSec (WAF) + ansible.builtin.import_tasks: appsec.yml + when: crowdsec_enabled | bool + tags: [crowdsec-configure] + +- name: Crowdsec | Manage the allowlist + ansible.builtin.import_tasks: allowlist.yml + when: crowdsec_enabled | bool + tags: [crowdsec-configure] + +- name: Crowdsec | Open or close the metrics port to the monitoring host + ansible.builtin.import_tasks: prometheus-metrics.yml + when: crowdsec_enabled | bool + tags: [crowdsec-configure] + +- name: Crowdsec | Configure OpenResty bouncer + ansible.builtin.import_tasks: configure-bouncer-openresty.yml + when: + - crowdsec_enabled | bool + - crowdsec_bouncer_enabled | bool + tags: [crowdsec-configure] + +# Switching off. crowdsec_bouncer_enabled=false removes the bouncer and keeps the engine. +# crowdsec_enabled=false removes both, but only from a host this role set up. +- name: Crowdsec | Check for a role-managed install + ansible.builtin.stat: + path: "{{ crowdsec_managed_marker }}" + register: crowdsec_managed + when: not (crowdsec_enabled | bool) + tags: [crowdsec-configure, crowdsec-remove] + +- name: Crowdsec | Remove OpenResty bouncer + ansible.builtin.import_tasks: remove-bouncer-openresty.yml + when: >- + (crowdsec_enabled | bool and not (crowdsec_bouncer_enabled | bool)) + or (not (crowdsec_enabled | bool) and crowdsec_managed.stat.exists | default(false)) + tags: [crowdsec-configure, crowdsec-remove] + +- name: Crowdsec | Remove Security Engine + ansible.builtin.import_tasks: remove-engine.yml + when: + - not (crowdsec_enabled | bool) + - crowdsec_managed.stat.exists | default(false) + tags: [crowdsec-remove] diff --git a/deploy/roles/crowdsec/tasks/install-engine.yml b/deploy/roles/crowdsec/tasks/install-engine.yml new file mode 100644 index 0000000..c0c6648 --- /dev/null +++ b/deploy/roles/crowdsec/tasks/install-engine.yml @@ -0,0 +1,233 @@ +--- +- name: Crowdsec | Install apt prerequisites for the CrowdSec repo + ansible.builtin.apt: + name: ca-certificates + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Crowdsec | Ensure the apt keyrings directory exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + owner: root + group: root + mode: "0755" + +# get_url downloads to a temp file and moves it into place only on success, so a failed +# download leaves nothing that a later run could mistake for the key. apt reads the +# ASCII-armored key directly because of the .asc extension. +- name: Crowdsec | Download CrowdSec signing key + ansible.builtin.get_url: + url: https://packagecloud.io/crowdsec/crowdsec/gpgkey + dest: "{{ crowdsec_keyring }}" + owner: root + group: root + mode: "0644" + register: crowdsec_key_download + until: crowdsec_key_download is succeeded + retries: 3 + delay: 5 + +# Hosts set up with the key dearmored into /usr/share/keyrings: apt refuses two entries for +# the same repo with different Signed-By values, and a failed curl there left a 0-byte +# keyring that apt rejects on every update. +- name: Crowdsec | Remove the repo entry signed by the dearmored keyring + ansible.builtin.apt_repository: + repo: "deb [signed-by={{ crowdsec_legacy_keyring }}] {{ crowdsec_repo_url }} any main" + state: absent + filename: crowdsec + update_cache: false + +- name: Crowdsec | Remove the dearmored keyring + ansible.builtin.file: + path: "{{ crowdsec_legacy_keyring }}" + state: absent + +- name: Crowdsec | Add CrowdSec official repository + ansible.builtin.apt_repository: + repo: "deb [signed-by={{ crowdsec_keyring }}] {{ crowdsec_repo_url }} any main" + state: present + filename: crowdsec + +# The crowdsec package's postinst runs debconf-driven prompts (crowdsec/lapi, crowdsec/capi) +# the first time it configures. Preseeding avoids it hanging/defaulting unpredictably under +# DEBIAN_FRONTEND=noninteractive (which the apt module already sets). +- name: Crowdsec | Preseed LAPI debconf question (always run a local API here) + ansible.builtin.debconf: + name: crowdsec + question: crowdsec/lapi + vtype: boolean + value: "true" + +- name: Crowdsec | Preseed Central API / community blocklist debconf question + ansible.builtin.debconf: + name: crowdsec + question: crowdsec/capi + vtype: boolean + value: "{{ crowdsec_online_api_enabled | bool | lower }}" + +- name: Crowdsec | Install CrowdSec Security Engine + ansible.builtin.apt: + name: "{{ 'crowdsec' if crowdsec_version | string == 'latest' else 'crowdsec=' ~ (crowdsec_version | string) }}" + state: "{{ 'latest' if crowdsec_version | string == 'latest' else 'present' }}" + allow_downgrade: true + allow_change_held_packages: true + update_cache: true + cache_valid_time: 3600 + +# The proxy role runs `apt upgrade` on every deploy, which would otherwise move a pinned +# engine to the newest version in the repo. +- name: Crowdsec | Hold the Security Engine at the pinned version + ansible.builtin.dpkg_selections: + name: crowdsec + selection: "{{ 'install' if crowdsec_version | string == 'latest' else 'hold' }}" + +- name: Crowdsec | Template local config overrides (LAPI bind address, Prometheus metrics) + ansible.builtin.template: + src: config.yaml.local.j2 + dest: "{{ crowdsec_managed_marker }}" + owner: root + group: root + mode: "0640" + notify: Restart crowdsec + +# The postinst writes the agent's LAPI URL once, from the default listen_uri, before the +# template above runs. Keep it in step with crowdsec_lapi_port, or the engine can't reach +# its own LAPI and crash-loops. +- name: Crowdsec | Point the engine's agent at the local API + ansible.builtin.lineinfile: + path: /etc/crowdsec/local_api_credentials.yaml + regexp: "^url:" + line: "url: {{ crowdsec_lapi_url }}" + diff: false + notify: Restart crowdsec + +# The crowdsec/capi debconf answer is only read on first install. An empty credentials +# file is how the engine runs with the Central API off ("push and pull to Central API +# disabled"); the package's own postinst leaves it that way when crowdsec/capi is false. +- name: Crowdsec | Check Central API credentials + ansible.builtin.stat: + path: /etc/crowdsec/online_api_credentials.yaml + register: crowdsec_capi_credentials + +- name: Crowdsec | Register with the Central API + ansible.builtin.command: cscli capi register --error + when: + - crowdsec_online_api_enabled | bool + - crowdsec_capi_credentials.stat.size | default(0) == 0 + changed_when: true + notify: Restart crowdsec + +- name: Crowdsec | Drop Central API credentials (no signal sharing, no community blocklist) + ansible.builtin.copy: + content: "" + dest: /etc/crowdsec/online_api_credentials.yaml + owner: root + group: root + mode: "0600" + diff: false + when: not (crowdsec_online_api_enabled | bool) + notify: Restart crowdsec + +# A conf.d file rather than a line in the proxy role's nginx.conf, so --tags crowdsec on an +# existing deployment sets it up too. nginx.conf includes conf.d/*.conf inside http {}. +- name: Crowdsec | Write the OpenResty access log CrowdSec reads + ansible.builtin.template: + src: openresty-access-log.conf.j2 + dest: "{{ crowdsec_openresty_conf_dir }}/crowdsec_access_log.conf" + owner: root + group: root + mode: "0640" + when: proxy == 'openresty' + notify: Crowdsec | Reload OpenResty + +# The postinst (cscli setup unattended) generates setup.openresty.yaml with a logs/*.log +# glob, which also tails the tab-separated perf.log that the nginx parser can't read. +# Once a hand-managed file exists in acquis.d, later unattended runs (on every package +# upgrade) stop without regenerating theirs. +- name: Crowdsec | Template OpenResty log acquisition + ansible.builtin.template: + src: acquis-openresty.yaml.j2 + dest: /etc/crowdsec/acquis.d/openresty.yaml + owner: root + group: root + mode: "0640" + when: proxy == 'openresty' + notify: Restart crowdsec + +# A second file reading the same logs makes the engine count every request twice, so +# scenarios ban at half their thresholds. Match on content, not on the generated file's +# name, which differs between crowdsec releases. +- name: Crowdsec | Find other acquisition files reading the OpenResty logs + ansible.builtin.find: + paths: + - /etc/crowdsec + - /etc/crowdsec/acquis.d + patterns: ["*.yaml", "*.yml"] + # re.match per line, so anchored at the line start + contains: ".*{{ crowdsec_openresty_logs_dir | regex_escape }}" + register: crowdsec_openresty_acquis + when: proxy == 'openresty' + +- name: Crowdsec | Remove generated duplicates of the OpenResty log acquisition + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: "{{ crowdsec_openresty_acquis_extra | select('search', '/setup\\.[^/]+$') | list }}" + vars: + crowdsec_openresty_acquis_extra: "{{ crowdsec_openresty_acquis.files | map(attribute='path') | reject('equalto', '/etc/crowdsec/acquis.d/openresty.yaml') | list }}" + when: proxy == 'openresty' + notify: Restart crowdsec + +- name: Crowdsec | Assert no other acquisition file reads the OpenResty logs + ansible.builtin.assert: + that: + - crowdsec_openresty_acquis_extra | reject('search', '/setup\\.[^/]+$') | list | length == 0 + fail_msg: >- + {{ crowdsec_openresty_acquis_extra | reject('search', '/setup\\.[^/]+$') | join(', ') }} + also read {{ crowdsec_openresty_logs_dir }}, which /etc/crowdsec/acquis.d/openresty.yaml + already covers. Every request would be counted twice. Remove those entries. + quiet: true + vars: + crowdsec_openresty_acquis_extra: "{{ crowdsec_openresty_acquis.files | map(attribute='path') | reject('equalto', '/etc/crowdsec/acquis.d/openresty.yaml') | list }}" + when: proxy == 'openresty' + +- name: Crowdsec | Create the systemd drop-in directory + ansible.builtin.file: + path: "{{ crowdsec_systemd_dropin | dirname }}" + state: directory + owner: root + group: root + mode: "0750" + when: crowdsec_memory_max | length > 0 + +- name: Crowdsec | Template the systemd memory limit + ansible.builtin.template: + src: systemd-override.conf.j2 + dest: "{{ crowdsec_systemd_dropin }}" + owner: root + group: root + mode: "0640" + when: crowdsec_memory_max | length > 0 + notify: Restart crowdsec + +- name: Crowdsec | Remove the systemd memory limit + ansible.builtin.file: + path: "{{ crowdsec_systemd_dropin }}" + state: absent + when: crowdsec_memory_max | length == 0 + notify: Restart crowdsec + +- name: Crowdsec | Ensure Security Engine is running and enabled + ansible.builtin.systemd: + name: crowdsec + state: started + enabled: true + daemon_reload: true + +# Bouncer packages auto-register against the LAPI during their own postinst (see +# configure-bouncer-*.yml) and need it reachable at that point, so config overrides +# must be live before we get there. +- name: Crowdsec | Flush handlers so config overrides apply before the bouncer installs + ansible.builtin.meta: flush_handlers diff --git a/deploy/roles/crowdsec/tasks/main.yml b/deploy/roles/crowdsec/tasks/main.yml new file mode 100644 index 0000000..f5a10cf --- /dev/null +++ b/deploy/roles/crowdsec/tasks/main.yml @@ -0,0 +1,50 @@ +--- +# Value checks (bouncer mode, remediation types, ...) live in meta/argument_specs.yml. +# These cover only what argument_specs can't express. roles/pre-install/tasks/main.yml +# runs the same checks before anything is deployed; these catch runs that skip that +# play (--limit). +- name: Crowdsec | Pre-flight | Assert supported connection type + ansible.builtin.assert: + that: + - ansible_connection | default('lxd') == 'lxd' + fail_msg: "crowdsec role only supports ansible_connection=lxd." + when: crowdsec_enabled | bool + tags: [always] + +# The bouncer is OpenResty-only. crowdsec-nginx-bouncer hard-depends on Ubuntu's +# libnginx-mod-http-lua, which needs the virtual package nginx-abi-1.24.0-1 - only Ubuntu's +# own nginx-core provides it. The nginx.org package the proxy role installs provides no +# nginx-abi at all, and nginx.org ships no lua module, so apt can never resolve it. +- name: Crowdsec | Pre-flight | Assert the bouncer runs on OpenResty + ansible.builtin.assert: + that: + - proxy == 'openresty' or not (crowdsec_bouncer_enabled | bool) + fail_msg: >- + The CrowdSec bouncer requires proxy=openresty (current: proxy={{ proxy }}). + nginx.org packages can't load the lua module crowdsec-nginx-bouncer depends on, + and the apache2 bouncer isn't implemented. Set proxy=openresty, or set + crowdsec_bouncer_enabled=false to run the Security Engine alone (detection only, + no blocking). + when: crowdsec_enabled | bool + tags: [always] + +# Without keys the bouncer fails to load its captcha plugin and serves every captcha +# decision as a ban. +- name: Crowdsec | Pre-flight | Assert captcha keys are set when captcha is enforced + ansible.builtin.assert: + that: + - >- + (crowdsec_bouncer_bouncing_on_type == 'ban' and crowdsec_bouncer_fallback_remediation == 'ban') + or (crowdsec_bouncer_captcha_site_key | length > 0 and crowdsec_bouncer_captcha_secret_key | length > 0) + fail_msg: >- + crowdsec_bouncer_bouncing_on_type={{ crowdsec_bouncer_bouncing_on_type }} and + crowdsec_bouncer_fallback_remediation={{ crowdsec_bouncer_fallback_remediation }} + need crowdsec_bouncer_captcha_site_key and crowdsec_bouncer_captcha_secret_key. + Set both, or set both options to ban. + when: + - crowdsec_enabled | bool + - crowdsec_bouncer_enabled | bool + tags: [always] + +- name: Crowdsec | Deploy or remove + ansible.builtin.import_tasks: deploy.yml diff --git a/deploy/roles/crowdsec/tasks/prometheus-metrics.yml b/deploy/roles/crowdsec/tasks/prometheus-metrics.yml new file mode 100644 index 0000000..e9b0342 --- /dev/null +++ b/deploy/roles/crowdsec/tasks/prometheus-metrics.yml @@ -0,0 +1,12 @@ +--- +# The rule is deleted again when metrics stop being exposed, so turning +# crowdsec_metrics_expose off (or removing the role) doesn't leave the port open. +- name: Crowdsec | Allow metrics scrapes from the monitoring host(s) only + community.general.ufw: + rule: allow + port: "{{ crowdsec_metrics_port | string }}" + src: "{{ hostvars[item]['service_ip'] | default(hostvars[item]['ansible_host']) }}" + proto: tcp + comment: "Allow crowdsec metrics scrape from prometheus/grafana server" + delete: "{{ not (crowdsec_metrics_exposed | bool) }}" + loop: "{{ groups['monitoring'] | default([]) }}" diff --git a/deploy/roles/crowdsec/tasks/remove-bouncer-openresty.yml b/deploy/roles/crowdsec/tasks/remove-bouncer-openresty.yml new file mode 100644 index 0000000..bb104d1 --- /dev/null +++ b/deploy/roles/crowdsec/tasks/remove-bouncer-openresty.yml @@ -0,0 +1,30 @@ +--- +# The postinst registered the bouncer with the LAPI and kept the API key in the package's +# conffile. Delete the registration and purge the conffile together, so re-enabling +# registers a new key instead of reusing one the LAPI no longer accepts. +- name: Crowdsec | List LAPI bouncer registrations + ansible.builtin.command: cscli bouncers list -o json + register: crowdsec_bouncer_registrations + changed_when: false + failed_when: false + +- name: Crowdsec | Delete the OpenResty bouncer's LAPI registrations + ansible.builtin.command: + argv: [cscli, bouncers, delete, "{{ item }}"] + loop: >- + {{ ((crowdsec_bouncers | selectattr('type', 'equalto', 'crowdsec-openresty-bouncer') | map(attribute='name') | list) + + (crowdsec_bouncers | selectattr('name', 'match', '^crowdsec-openresty-bouncer') | map(attribute='name') | list)) + | unique }} + vars: + crowdsec_bouncers: >- + {{ (crowdsec_bouncer_registrations.stdout | from_json | default([], true)) + if crowdsec_bouncer_registrations.rc == 0 else [] }} + changed_when: true + +# Removes crowdsec_openresty.conf from conf.d/ along with the lua code. +- name: Crowdsec | Purge the OpenResty bouncer package + ansible.builtin.apt: + name: crowdsec-openresty-bouncer + state: absent + purge: true + notify: Crowdsec | Reload OpenResty diff --git a/deploy/roles/crowdsec/tasks/remove-engine.yml b/deploy/roles/crowdsec/tasks/remove-engine.yml new file mode 100644 index 0000000..4372743 --- /dev/null +++ b/deploy/roles/crowdsec/tasks/remove-engine.yml @@ -0,0 +1,43 @@ +--- +- name: Crowdsec | Remove the OpenResty access log CrowdSec reads + ansible.builtin.file: + path: "{{ crowdsec_openresty_conf_dir }}/crowdsec_access_log.conf" + state: absent + notify: Crowdsec | Reload OpenResty + +- name: Crowdsec | Close the metrics port + ansible.builtin.import_tasks: prometheus-metrics.yml + vars: + crowdsec_metrics_exposed: false + +# Not purged: /etc/crowdsec and /var/lib/crowdsec keep the machine and Central API +# credentials and local decisions, so re-enabling picks up where it left off. +# `apt purge crowdsec` wipes them. +- name: Crowdsec | Remove the systemd memory limit drop-in + ansible.builtin.file: + path: "{{ crowdsec_systemd_dropin }}" + state: absent + +- name: Crowdsec | Remove the Security Engine package + ansible.builtin.apt: + name: crowdsec + state: absent + allow_change_held_packages: true + +- name: Crowdsec | Remove the CrowdSec apt repository + ansible.builtin.apt_repository: + repo: "deb [signed-by={{ crowdsec_keyring }}] {{ crowdsec_repo_url }} any main" + state: absent + filename: crowdsec + +- name: Crowdsec | Remove the CrowdSec signing key + ansible.builtin.file: + path: "{{ crowdsec_keyring }}" + state: absent + +# Last, so an interrupted removal is retried on the next run. Without the marker, later +# runs with crowdsec_enabled=false skip removal. +- name: Crowdsec | Remove the role-managed marker + ansible.builtin.file: + path: "{{ crowdsec_managed_marker }}" + state: absent diff --git a/deploy/roles/crowdsec/templates/acquis-appsec.yaml.j2 b/deploy/roles/crowdsec/templates/acquis-appsec.yaml.j2 new file mode 100644 index 0000000..72a8309 --- /dev/null +++ b/deploy/roles/crowdsec/templates/acquis-appsec.yaml.j2 @@ -0,0 +1,9 @@ +{{ ansible_managed | comment }} +# AppSec listener the OpenResty bouncer forwards each request to (APPSEC_URL). +source: appsec +listen_addr: "127.0.0.1:{{ crowdsec_appsec_port }}" +appsec_configs: + - crowdsecurity/appsec-default + - dhis2/body-limits +labels: + type: appsec diff --git a/deploy/roles/crowdsec/templates/acquis-openresty.yaml.j2 b/deploy/roles/crowdsec/templates/acquis-openresty.yaml.j2 new file mode 100644 index 0000000..9454fb3 --- /dev/null +++ b/deploy/roles/crowdsec/templates/acquis-openresty.yaml.j2 @@ -0,0 +1,7 @@ +{{ ansible_managed | comment }} +filenames: + - {{ crowdsec_openresty_logs_dir }}/access.log + - {{ crowdsec_openresty_logs_dir }}/error.log +labels: + type: nginx +source: file diff --git a/deploy/roles/crowdsec/templates/appsec-body-limits.yaml.j2 b/deploy/roles/crowdsec/templates/appsec-body-limits.yaml.j2 new file mode 100644 index 0000000..5461764 --- /dev/null +++ b/deploy/roles/crowdsec/templates/appsec-body-limits.yaml.j2 @@ -0,0 +1,9 @@ +{{ ansible_managed | comment }} +# Loaded after crowdsecurity/appsec-default (see acquis.d/appsec.yaml). By default AppSec +# blocks every request whose body is over 10 MiB, raising an alert against the sender's +# IP, which would stop DHIS2 data and metadata imports. +name: dhis2/body-limits +on_load: + - apply: + - SetMaxBodySize({{ crowdsec_appsec_max_body_size | int }}) + - SetBodySizeExceededAction("{{ crowdsec_appsec_body_size_exceeded_action }}") diff --git a/deploy/roles/crowdsec/templates/config.yaml.local.j2 b/deploy/roles/crowdsec/templates/config.yaml.local.j2 new file mode 100644 index 0000000..5ee27da --- /dev/null +++ b/deploy/roles/crowdsec/templates/config.yaml.local.j2 @@ -0,0 +1,11 @@ +{{ ansible_managed | comment }} +# Overrides for /etc/crowdsec/config.yaml, merged in by crowdsec's own .local patcher +# (pkg/csconfig: csyaml.NewPatcher(configFile, ".local")). Left in place across +# `apt upgrade` of the crowdsec package, unlike editing config.yaml directly. +api: + server: + listen_uri: "127.0.0.1:{{ crowdsec_lapi_port }}" +prometheus: + enabled: {{ crowdsec_metrics_enabled | bool | lower }} + listen_addr: "{{ crowdsec_metrics_listen_addr }}" + listen_port: {{ crowdsec_metrics_port }} diff --git a/deploy/roles/crowdsec/templates/openresty-access-log.conf.j2 b/deploy/roles/crowdsec/templates/openresty-access-log.conf.j2 new file mode 100644 index 0000000..7023b2c --- /dev/null +++ b/deploy/roles/crowdsec/templates/openresty-access-log.conf.j2 @@ -0,0 +1,4 @@ +{{ ansible_managed | comment }} +# CrowdSec's nginx parser reads the combined layout, not the tab-separated perf.log. +# Buffered: one write per 64k instead of per request; CrowdSec sees a request up to 5 s late. +access_log {{ crowdsec_openresty_logs_dir }}/access.log combined buffer=64k flush=5s; diff --git a/deploy/roles/crowdsec/templates/systemd-override.conf.j2 b/deploy/roles/crowdsec/templates/systemd-override.conf.j2 new file mode 100644 index 0000000..b6b8cee --- /dev/null +++ b/deploy/roles/crowdsec/templates/systemd-override.conf.j2 @@ -0,0 +1,3 @@ +{{ ansible_managed | comment }} +[Service] +MemoryMax={{ crowdsec_memory_max }} diff --git a/deploy/roles/crowdsec/vars/main.yml b/deploy/roles/crowdsec/vars/main.yml new file mode 100644 index 0000000..5fc807a --- /dev/null +++ b/deploy/roles/crowdsec/vars/main.yml @@ -0,0 +1,28 @@ +--- +# Role-internal constants and derived values - not for override. +crowdsec_lapi_url: "http://127.0.0.1:{{ crowdsec_lapi_port }}" +crowdsec_appsec_url: "http://127.0.0.1:{{ crowdsec_appsec_port }}" +crowdsec_appsec_active: "{{ crowdsec_appsec_enabled | bool and crowdsec_bouncer_enabled | bool }}" + +crowdsec_metrics_exposed: "{{ crowdsec_metrics_enabled | bool and crowdsec_metrics_expose | bool }}" +crowdsec_metrics_listen_addr: "{{ (service_ip | default(ansible_host)) if crowdsec_metrics_exposed | bool else '127.0.0.1' }}" + +crowdsec_repo_url: "https://packagecloud.io/crowdsec/crowdsec/any" +crowdsec_keyring: "/etc/apt/keyrings/crowdsec.asc" +# Where the key used to be dearmored to with curl | gpg; see install-engine.yml. +crowdsec_legacy_keyring: "/usr/share/keyrings/crowdsec-archive-keyring.gpg" + +crowdsec_openresty_conf_dir: "/usr/local/openresty/nginx/conf/conf.d" +crowdsec_openresty_logs_dir: "/usr/local/openresty/nginx/logs" +crowdsec_bouncer_conf: "/etc/crowdsec/bouncers/crowdsec-openresty-bouncer.conf" + +# Templated by this role only, so its presence marks a role-managed install that +# crowdsec_enabled=false may remove. +crowdsec_managed_marker: "/etc/crowdsec/config.yaml.local" + +crowdsec_systemd_dropin: "/etc/systemd/system/crowdsec.service.d/override.conf" + +crowdsec_allowlist_name: "dhis2" +crowdsec_appsec_collections: + - crowdsecurity/appsec-virtual-patching + - crowdsecurity/appsec-generic-rules diff --git a/deploy/roles/monitoring/templates/scrape-configs.j2 b/deploy/roles/monitoring/templates/scrape-configs.j2 index 43bc6eb..2d3179b 100644 --- a/deploy/roles/monitoring/templates/scrape-configs.j2 +++ b/deploy/roles/monitoring/templates/scrape-configs.j2 @@ -20,3 +20,12 @@ - targets: - {{ lxd_network.split('/')[0] }}:9100 {% endif %} +{% for web_host in groups['web'] %} +{# crowdsec_metrics_expose defaults to true in the crowdsec role whenever this template is used #} +{% if hostvars[web_host]['crowdsec_enabled'] | default(false) | bool and hostvars[web_host]['crowdsec_metrics_enabled'] | default(true) | bool and hostvars[web_host]['crowdsec_metrics_expose'] | default(true) | bool %} + - job_name: crowdsec-{{ web_host }} + static_configs: + - targets: + - {{ hostvars[web_host]['service_ip'] | default(hostvars[web_host]['ansible_host']) }}:{{ hostvars[web_host]['crowdsec_metrics_port'] | default(6060) }} +{% endif %} +{% endfor %} diff --git a/deploy/roles/pre-install/tasks/main.yml b/deploy/roles/pre-install/tasks/main.yml index b74c581..a8567ba 100644 --- a/deploy/roles/pre-install/tasks/main.yml +++ b/deploy/roles/pre-install/tasks/main.yml @@ -1,4 +1,30 @@ --- +# Same checks as roles/crowdsec/tasks/main.yml, which only runs at the end of the proxy +# role, after postgres and the proxy are already deployed. Failing here stops the run +# before anything changes. The defaults mirror roles/crowdsec/defaults/main.yml. +- name: Pre-flight | Assert CrowdSec settings on [web] hosts + ansible.builtin.assert: + that: + - hostvars[item]['ansible_connection'] | default('lxd') == 'lxd' + - >- + hostvars[item]['proxy'] | default('') == 'openresty' + or not (hostvars[item]['crowdsec_bouncer_enabled'] | default(true) | bool) + - >- + not (hostvars[item]['crowdsec_bouncer_enabled'] | default(true) | bool) + or (hostvars[item]['crowdsec_bouncer_bouncing_on_type'] | default('ban') == 'ban' + and hostvars[item]['crowdsec_bouncer_fallback_remediation'] | default('ban') == 'ban') + or (hostvars[item]['crowdsec_bouncer_captcha_site_key'] | default('') | length > 0 + and hostvars[item]['crowdsec_bouncer_captcha_secret_key'] | default('') | length > 0) + fail_msg: >- + {{ item }}: crowdsec_enabled=true needs ansible_connection=lxd, and the bouncer needs + proxy=openresty (or set crowdsec_bouncer_enabled=false for detection only). Captcha + remediation needs crowdsec_bouncer_captcha_site_key and + crowdsec_bouncer_captcha_secret_key. + quiet: true + loop: "{{ groups['web'] | default([]) }}" + when: hostvars[item]['crowdsec_enabled'] | default(false) | bool + tags: [always] + - name: Ensure secrets directory exists vars: ansible_connection: local diff --git a/deploy/roles/proxy/tasks/main.yml b/deploy/roles/proxy/tasks/main.yml index 5d1bfec..9a62dbd 100644 --- a/deploy/roles/proxy/tasks/main.yml +++ b/deploy/roles/proxy/tasks/main.yml @@ -16,3 +16,15 @@ - server_monitoring is defined - server_monitoring | trim == 'munin' tags: [securing-munin] + +# Not gated on crowdsec_enabled: with it false, the role removes CrowdSec from hosts it +# set up earlier and does nothing on the rest. +- name: Crowdsec | Install, configure or remove engine and bouncer + ansible.builtin.include_role: + name: crowdsec + apply: + tags: [crowdsec] + when: + - proxy is defined + - inventory_hostname in groups['web'] + tags: [crowdsec, crowdsec-install, crowdsec-configure, crowdsec-remove] diff --git a/docs/CrowdSec-WAF.md b/docs/CrowdSec-WAF.md new file mode 100644 index 0000000..66f12dd --- /dev/null +++ b/docs/CrowdSec-WAF.md @@ -0,0 +1,191 @@ +# CrowdSec WAF and IP banning + +[CrowdSec](https://www.crowdsec.net/) runs on the `[web]` host next to the reverse +proxy. It gives two layers of protection, both off by default: + +- **IP banning.** The Security Engine reads the proxy's access log and bans IPs + whose traffic matches an attack scenario (scanners, probing, path traversal, + brute force, bad user agents). With the Central API on, it also applies + CrowdSec's community blocklist. +- **WAF (AppSec).** The OpenResty bouncer sends every request to the engine's + AppSec listener, which blocks in-band on a virtual-patching rule (known CVEs) + or a generic rule (for example template injection), before the request + reaches DHIS2. + +The bouncer enforces both in OpenResty's access phase: a banned IP or a blocked +request gets a 403. + +## Requirements + +- `ansible_connection=lxd` (the single-server architecture). SSH deployments are + not supported. +- `proxy=openresty`. The bouncer is a lua module; the nginx.org packages the + proxy role installs cannot load it, and apache2 has no bouncer. With another + proxy, set `crowdsec_bouncer_enabled=false` to run the engine alone + (detection only, nothing is blocked). +- Tested on Ubuntu 24.04 and 22.04. + +## Enable + +In `deploy/inventory/hosts`, under `[web:vars]`: + +``` +crowdsec_enabled=true +``` + +Then run the playbook. On an existing server: + +``` +sudo ansible-playbook dhis2.yml --tags proxy-install +``` + +The run fails before changing anything if the settings are inconsistent (wrong +proxy, captcha without keys). + +### Check it works + +Inside the proxy container (`lxc exec proxy -- bash`): + +``` +cscli metrics show acquisition # access.log lines read and parsed +cscli bouncers list # one crowdsec-openresty-bouncer, last_pull a few seconds old +cscli decisions list # current bans +curl -sI https:///.env | head -1 # from outside: HTTP/2 403 (AppSec) +``` + +To see a ban end to end: + +``` +cscli decisions add -i -d 1m +# a request from gets 403 within 10 s +cscli decisions delete -i +``` + +The playbook itself asserts that the bouncer has pulled decisions from the +local API after each run, so a bouncer that silently lets everything through +fails the deployment. + +## Variables + +All under `[web:vars]`. Defaults in `deploy/roles/crowdsec/defaults/main.yml`; +full descriptions in `deploy/roles/crowdsec/meta/argument_specs.yml`. + +| Variable | Default | Notes | +| --- | --- | --- | +| `crowdsec_enabled` | `false` | Master switch. Setting it back to `false` removes CrowdSec from hosts this role set up. | +| `crowdsec_version` | `latest` | Pin to an exact package version to hold it across `apt upgrade`. | +| `crowdsec_online_api_enabled` | `true` | Central API: shares banned IPs and the scenarios they triggered with CrowdSec and pulls the community blocklist. See below. | +| `crowdsec_bouncer_enabled` | `true` | `false` keeps the engine (detection only). | +| `crowdsec_bouncer_mode` | `stream` | `stream` caches decisions, so bans hold while the engine restarts. `live` asks per request and fails open. | +| `crowdsec_bouncer_bouncing_on_type` | `ban` | `captcha` or `all` need the captcha keys. | +| `crowdsec_bouncer_captcha_provider` | `recaptcha` | `recaptcha`, `hcaptcha`, `turnstile`. | +| `crowdsec_bouncer_captcha_site_key` | `""` | | +| `crowdsec_bouncer_captcha_secret_key` | `""` | Put it in `host_vars//vault.yml`. | +| `crowdsec_appsec_enabled` | `true` | The WAF. Needs the bouncer. | +| `crowdsec_appsec_failure_action` | `passthrough` | What the bouncer does when AppSec does not answer. `deny` blocks. | +| `crowdsec_appsec_max_body_size` | `10485760` | Bytes. See "What AppSec inspects". | +| `crowdsec_appsec_body_size_exceeded_action` | `partial` | `partial`, `allow`, or `drop`. `drop` blocks imports over the limit. | +| `crowdsec_allowlist` | `[]` | IPs/CIDRs never inspected or banned. | +| `crowdsec_metrics_enabled` | `true` | Prometheus endpoint, loopback only unless exposed. | +| `crowdsec_metrics_expose` | auto | `true` when `server_monitoring` is Prometheus-based and a `[monitoring]` host exists. | +| `crowdsec_memory_max` | `1G` | systemd `MemoryMax` for the engine. Empty removes the limit. | + +## Central API and data sharing + +With `crowdsec_online_api_enabled=true` (the default) the engine registers with +CrowdSec's Central API and sends it the IPs it bans together with the scenario +names that triggered the ban. Nothing else leaves the server: no request +contents, no DHIS2 data, no logs. In return it pulls the community blocklist, +which is most of the IP-banning value. + +If that sharing is not acceptable, set: + +``` +crowdsec_online_api_enabled=false +``` + +Only locally detected attackers are banned then. + +## Allowlist + +Some networks put many legitimate users behind one address (a facility's +carrier-grade NAT, a national proxy). One user's behaviour must not ban them +all: + +``` +crowdsec_allowlist=["203.0.113.10", "198.51.100.0/24"] +``` + +Listed addresses are never inspected by AppSec and never banned, locally or by +the community blocklist. Adding an entry also lifts existing bans on it. + +## What AppSec inspects + +Requests are inspected in full up to `crowdsec_appsec_max_body_size` +(10 MiB). DHIS2 accepts imports up to 100 MB, so larger bodies are let through +rather than blocked: + +- URI, query string and headers are always inspected. +- Form bodies are inspected up to the limit. +- JSON bodies over the limit are not body-inspected. + +Raising the limit costs engine memory per in-flight request, and past about +50 MiB the bouncer's AppSec call times out and lets the request through +anyway. Keep the default unless you have a specific reason. + +AppSec fails open (`passthrough`): if the engine is down, requests go through. +Stream-mode bans still hold in the bouncer during an engine restart. + +## Prometheus and Grafana + +With `server_monitoring=grafana`, `prometheus` or `grafana/prometheus` and a +`[monitoring]` host, the engine's metrics endpoint binds to the host's service +IP, ufw lets only the monitoring host(s) in, and Prometheus gets a +`crowdsec-` scrape job. The endpoint also serves Go's `/debug/pprof`, +which is why it is never exposed wider than that. The generic `InstanceDown` +alert covers the job; there is no CrowdSec dashboard yet. + +With Munin (the default) there is no CrowdSec view; use `cscli metrics`. + +## Switching off + +| Setting | Effect | +| --- | --- | +| `crowdsec_bouncer_enabled=false` | Purges the bouncer and deletes its API registration. The engine keeps detecting; nothing is blocked. | +| `crowdsec_enabled=false` | Also removes the engine package, its apt repository, the metrics firewall rule and the proxy's CrowdSec log. Only on hosts this role set up. `/etc/crowdsec` and `/var/lib/crowdsec` are kept, so re-enabling reuses the registration. | + +## Known limitations + +- **Log rotation.** OpenResty's `access.log`, `error.log` and `perf.log` under + `/usr/local/openresty/nginx/logs/` are not rotated. CrowdSec adds the + combined `access.log`; watch the disk until rotation ships. +- **Patching.** `unattended-upgrades` only tracks Ubuntu repositories. CrowdSec + and the bouncer upgrade when the playbook runs. Hub rules (scenarios, + AppSec virtual patches) do update daily on their own, through the package's + `crowdsec-hubupdate.timer`. +- **HTTP only.** The engine reads the proxy's logs; it does not watch SSH or + other services on the host. + +## Troubleshooting + +**The run fails with "has not pulled decisions from the LAPI".** The bouncer +is installed but not enforcing. Check, in the proxy container: + +``` +dpkg --verify crowdsec-openresty-bouncer # crowdsec_openresty.conf must not be listed +grep ^API_KEY /etc/crowdsec/bouncers/crowdsec-openresty-bouncer.conf +cscli bouncers list # the key must belong to a listed bouncer +tail /usr/local/openresty/nginx/logs/error.log +``` + +A hand-edited `conf.d/crowdsec_openresty.conf` is the usual cause; reinstall +the bouncer package or run once with `crowdsec_bouncer_enabled=false` and then +back on, which purges it and registers a fresh key. + +**A legitimate user is banned.** `cscli decisions list` shows the reason. +`cscli decisions delete -i ` lifts it; add the address to +`crowdsec_allowlist` if it is a shared egress. + +**An import is blocked with 403.** `cscli alerts list` shows the AppSec rule. +Bodies over the limit are not blocked by default; a 403 on an import means a +rule matched the URI, a header or the first part of a form body. diff --git a/molecule/crowdsec/converge.yml b/molecule/crowdsec/converge.yml new file mode 100644 index 0000000..23aa2c4 --- /dev/null +++ b/molecule/crowdsec/converge.yml @@ -0,0 +1,9 @@ +--- +- name: Converge + hosts: web + become: true + tasks: + - name: Converge | Deploy CrowdSec + ansible.builtin.include_role: + name: crowdsec + tasks_from: deploy.yml diff --git a/molecule/crowdsec/molecule.yml b/molecule/crowdsec/molecule.yml new file mode 100644 index 0000000..fdaec4a --- /dev/null +++ b/molecule/crowdsec/molecule.yml @@ -0,0 +1,62 @@ +--- +# CrowdSec on OpenResty: install, idempotence, enforcement (verify.yml), then switching it +# off (side_effect.yml). The role's own pre-flight accepts only ansible_connection=lxd, so +# the playbooks drive tasks/deploy.yml - everything after those asserts - over docker. +# MOLECULE_CROWDSEC_IMAGE overrides the image, e.g. with one that has the packages cached. +dependency: + name: galaxy + options: + ignore-certs: true + ignore-errors: true +driver: + name: docker +platforms: + - name: crowdsec-web + image: ${MOLECULE_CROWDSEC_IMAGE:-geerlingguy/docker-ubuntu2404-ansible:latest} + pre_build_image: true + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + privileged: true + cgroupns_mode: host + groups: + - web +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ${MOLECULE_PROJECT_DIRECTORY}/deploy/roles + ANSIBLE_FILTER_PLUGINS: ${MOLECULE_PROJECT_DIRECTORY}/deploy/filter_plugins + ANSIBLE_LIBRARY: ${MOLECULE_PROJECT_DIRECTORY}/deploy/library + config_options: + defaults: + interpreter_python: auto_silent + callback_result_format: yaml + inventory: + hosts: + all: + children: + monitoring: + hosts: {} + instances: + hosts: {} + group_vars: + all: + proxy: openresty + crowdsec_enabled: true + # Keep CI runs from registering throwaway machines with CrowdSec's Central API. + crowdsec_online_api_enabled: false + crowdsec_allowlist: ["192.0.2.10"] +verifier: + name: ansible +scenario: + test_sequence: + - dependency + - destroy + - syntax + - create + - prepare + - converge + - idempotence + - verify + - side_effect + - destroy diff --git a/molecule/crowdsec/prepare.yml b/molecule/crowdsec/prepare.yml new file mode 100644 index 0000000..ec4f7ad --- /dev/null +++ b/molecule/crowdsec/prepare.yml @@ -0,0 +1,40 @@ +--- +# Stands in for the parts of the play that run before the crowdsec role: the proxy role's +# real OpenResty install, and a static-file default vhost in place of create-instance's +# DHIS2 vhosts. The bouncer runs in the access phase, so a vhost that answers with +# `return` (rewrite phase) would never reach it. +- name: Prepare + hosts: web + become: true + tasks: + - name: Prepare | Install ufw + ansible.builtin.apt: + name: ufw + state: present + update_cache: true + + - name: Prepare | Install OpenResty with the proxy role + ansible.builtin.include_role: + name: proxy + tasks_from: openresty-install.yml + + - name: Prepare | Static-file default vhost + ansible.builtin.copy: + dest: /usr/local/openresty/nginx/conf/conf.d/default.conf + content: | + server { + listen 80 default_server; + client_max_body_size 100M; + root /usr/local/openresty/nginx/html; + location / { try_files $uri $uri/ =404; } + } + owner: root + group: root + mode: "0640" + + - name: Prepare | Start OpenResty with the vhost + ansible.builtin.service: + name: openresty + state: restarted + enabled: true + changed_when: false diff --git a/molecule/crowdsec/side_effect.yml b/molecule/crowdsec/side_effect.yml new file mode 100644 index 0000000..7cbff9e --- /dev/null +++ b/molecule/crowdsec/side_effect.yml @@ -0,0 +1,107 @@ +--- +# Switching off: first the bouncer alone, then CrowdSec entirely. +- name: Side effect | Switch the bouncer off + hosts: web + become: true + gather_facts: false + tasks: + - name: Side effect | Deploy with crowdsec_bouncer_enabled=false + ansible.builtin.include_role: + name: crowdsec + tasks_from: deploy.yml + vars: + crowdsec_bouncer_enabled: false + + - name: Side effect | Apply pending reloads + ansible.builtin.meta: flush_handlers + + - name: Side effect | Read the bouncer package state + ansible.builtin.command: dpkg-query -W -f=${Status} crowdsec-openresty-bouncer + register: side_bouncer_pkg + changed_when: false + failed_when: false + + - name: Side effect | List bouncer registrations + ansible.builtin.command: cscli bouncers list -o json + register: side_bouncers + changed_when: false + + - name: Side effect | Check the bouncer's conf.d file + ansible.builtin.stat: + path: /usr/local/openresty/nginx/conf/conf.d/crowdsec_openresty.conf + register: side_bouncer_conf + + - name: Side effect | The bouncer is purged and unregistered; the engine keeps running + ansible.builtin.assert: + that: + - side_bouncer_pkg.stdout is not search('installed|config-files') + - side_bouncers.stdout | from_json | default([], true) | length == 0 + - not side_bouncer_conf.stat.exists + + - name: Side effect | The engine is still running + ansible.builtin.command: systemctl is-active crowdsec openresty + changed_when: false + + - name: Side effect | Nothing blocks without the bouncer + ansible.builtin.uri: + url: http://127.0.0.1/.env + status_code: 404 + +- name: Side effect | Switch CrowdSec off + hosts: web + become: true + gather_facts: false + tasks: + - name: Side effect | Deploy with crowdsec_enabled=false + ansible.builtin.include_role: + name: crowdsec + tasks_from: deploy.yml + vars: + crowdsec_enabled: false + + - name: Side effect | Apply pending reloads + ansible.builtin.meta: flush_handlers + + - name: Side effect | Read the engine package state + ansible.builtin.command: dpkg-query -W -f=${Status} crowdsec + register: side_engine_pkg + changed_when: false + failed_when: false + + - name: Side effect | Check the files the role leaves behind or removes + ansible.builtin.stat: + path: "{{ item }}" + loop: + - /etc/crowdsec/config.yaml.local + - /usr/local/openresty/nginx/conf/conf.d/crowdsec_access_log.conf + - /etc/apt/keyrings/crowdsec.asc + - /etc/apt/sources.list.d/crowdsec.list + register: side_files + + - name: Side effect | Engine, marker, access log, key and repo are gone + ansible.builtin.assert: + that: + - side_engine_pkg.stdout is not search('ok installed') + - side_files.results | selectattr('stat.exists') | list | length == 0 + + - name: Side effect | OpenResty config is valid and still serving + ansible.builtin.command: openresty -t + changed_when: false + + - name: Side effect | A normal request is served + ansible.builtin.uri: + url: http://127.0.0.1/ + status_code: 200 + +# Removal must be a no-op once done: the marker is gone, so the role does nothing. +- name: Side effect | Rerun with CrowdSec off + hosts: web + become: true + gather_facts: false + tasks: + - name: Side effect | Deploy with crowdsec_enabled=false again + ansible.builtin.include_role: + name: crowdsec + tasks_from: deploy.yml + vars: + crowdsec_enabled: false diff --git a/molecule/crowdsec/verify.yml b/molecule/crowdsec/verify.yml new file mode 100644 index 0000000..9c67e93 --- /dev/null +++ b/molecule/crowdsec/verify.yml @@ -0,0 +1,143 @@ +--- +- name: Verify + hosts: web + become: true + gather_facts: false + tasks: + - name: Verify | OpenResty config is valid + ansible.builtin.command: openresty -t + changed_when: false + + - name: Verify | CrowdSec and OpenResty are running + ansible.builtin.command: systemctl is-active crowdsec openresty + changed_when: false + + - name: Verify | List bouncer registrations + ansible.builtin.command: cscli bouncers list -o json + register: verify_bouncers + changed_when: false + + - name: Verify | Exactly one OpenResty bouncer is registered + ansible.builtin.assert: + that: + - verify_bouncers.stdout | from_json | selectattr('name', 'match', '^crowdsec-openresty-bouncer') | list | length == 1 + + - name: Verify | A normal request is served + ansible.builtin.uri: + url: http://127.0.0.1/ + status_code: 200 + + - name: Verify | Read the bouncer config mode + ansible.builtin.stat: + path: /etc/crowdsec/bouncers/crowdsec-openresty-bouncer.conf + register: verify_bouncer_conf + + - name: Verify | The bouncer config (API key) is root-only + ansible.builtin.assert: + that: + - verify_bouncer_conf.stat.mode == '0600' + - verify_bouncer_conf.stat.pw_name == 'root' + + - name: Verify | Read the engine's memory limit + ansible.builtin.command: systemctl show crowdsec -p MemoryMax --value + register: verify_memory_max + changed_when: false + + - name: Verify | The engine has a memory ceiling + ansible.builtin.assert: + that: + - verify_memory_max.stdout != 'infinity' + + # A decision made by hand proves the bouncer's stream path end to end: pull from the + # LAPI, cache, enforce in the access phase. AppSec blocks below don't exercise it. + - name: Verify | Enforce a manual ban through the bouncer + block: + - name: Verify | Ban the loopback client + ansible.builtin.command: cscli decisions add -i 127.0.0.1 -d 2m --reason molecule + changed_when: true + + - name: Verify | The banned client gets 403 within one stream pull + ansible.builtin.uri: + url: http://127.0.0.1/ + status_code: 403 + register: verify_ban + retries: 15 + delay: 2 + until: verify_ban.status == 403 + always: + - name: Verify | Lift the manual ban + ansible.builtin.command: cscli decisions delete -i 127.0.0.1 + changed_when: true + + - name: Verify | The client is served again + ansible.builtin.uri: + url: http://127.0.0.1/ + status_code: 200 + register: verify_unban + retries: 15 + delay: 2 + until: verify_unban.status == 200 + + - name: Verify | AppSec blocks a virtual-patching match + ansible.builtin.uri: + url: http://127.0.0.1/.env + status_code: 403 + + - name: Verify | Create a 12 MiB body, over the AppSec inspection limit + ansible.builtin.command: + cmd: truncate -s 12M /tmp/verify-12m.bin + creates: /tmp/verify-12m.bin + + # partial mode: inspected up to the limit and let through, so the static vhost answers + # the POST (404/405) instead of the bouncer's 403. + - name: Verify | A large import-sized body isn't blocked + ansible.builtin.uri: + url: http://127.0.0.1/api/dataValueSets + method: POST + src: /tmp/verify-12m.bin + remote_src: true + status_code: [404, 405] + + - name: Verify | access.log is read and parsed + ansible.builtin.command: cscli metrics show acquisition -o json + register: verify_acquisition + changed_when: false + until: >- + ((verify_acquisition.stdout | from_json).acquisition + | default({}))['file:/usr/local/openresty/nginx/logs/access.log']['parsed'] | default(0) | int > 0 + retries: 10 + delay: 2 + + - name: Verify | Find acquisition files reading the OpenResty logs + ansible.builtin.find: + paths: [/etc/crowdsec, /etc/crowdsec/acquis.d] + patterns: ["*.yaml", "*.yml"] + contains: '.*/usr/local/openresty/nginx/logs' + register: verify_openresty_acquis + + - name: Verify | Only the role's acquisition file reads them + ansible.builtin.assert: + that: + - verify_openresty_acquis.files | map(attribute='path') | list == ['/etc/crowdsec/acquis.d/openresty.yaml'] + + - name: Verify | Read the allowlist + ansible.builtin.command: cscli allowlists inspect dhis2 -o json + register: verify_allowlist + changed_when: false + + - name: Verify | The allowlist holds crowdsec_allowlist + ansible.builtin.assert: + that: + - (verify_allowlist.stdout | from_json)['items'] | map(attribute='value') | list == ['192.0.2.10'] + + # No [monitoring] host here, so crowdsec_metrics_expose defaults to false. + - name: Verify | Metrics listen on loopback only + ansible.builtin.command: ss -Hltn sport = :6060 + register: verify_metrics_listen + changed_when: false + + - name: Verify | Metrics aren't on a routable address + ansible.builtin.assert: + that: + - verify_metrics_listen.stdout is search('127\.0\.0\.1:6060') + - verify_metrics_listen.stdout is not search('0\.0\.0\.0:6060')