Add macros to grant roles access to schemas and tables

dbt_project/analyses/grants/_analysis.yml

@@ -0,0 +1,28 @@
+version: 2
+
+analyses:
+  - name: check_schema_role_privileges
+    description: |
+      Lists schema-level privileges for roles across non-system schemas.
+      Useful for validating whether grant macros are correctly applying
+      CREATE and USAGE permissions.
+
+      Optional vars:
+        - grants_schema_like (default: %brainpower%)
+        - grants_role_like (default: %include%)
+
+      Example dbt show:
+        dbt show --select path:analyses/grants/check_schema_role_privileges.sql
+
+      Example dbt show with vars:
+        dbt show --select path:analyses/grants/check_schema_role_privileges.sql --vars '{"grants_schema_like":"dev_inc_%","grants_role_like":"include%"}'
+
+    columns:
+      - name: schema_name
+        description: Name of the schema being inspected.
+      - name: role_name
+        description: Name of the role being inspected.
+      - name: can_create
+        description: YES when the role has CREATE on the schema; otherwise NO.
+      - name: can_usage
+        description: YES when the role has USAGE on the schema; otherwise NO.

dbt_project/analyses/grants/check_schema_role_privileges.sql

@@ -0,0 +1,30 @@
+-- PostgreSQL: List roles and their privileges on each schema
+-- Optional vars:
+--   grants_schema_like (default: %brainpower%)
+--   grants_role_like   (default: %include%)
+
+{% set grants_schema_like = var('grants_schema_like', '%brainpower%') %}
+{% set grants_role_like = var('grants_role_like', '%include%') %}
+
+SELECT
+    n.nspname AS schema_name,
+    r.rolname AS role_name,
+    CASE
+        WHEN has_schema_privilege(r.rolname, n.nspname, 'CREATE') THEN 'YES'
+        ELSE 'NO'
+    END AS can_create,
+    CASE
+        WHEN has_schema_privilege(r.rolname, n.nspname, 'USAGE') THEN 'YES'
+        ELSE 'NO'
+    END AS can_usage
+FROM
+    pg_namespace n
+CROSS JOIN
+    pg_roles r
+WHERE
+    n.nspname NOT LIKE 'pg_%'
+    AND n.nspname <> 'information_schema'
+    AND n.nspname LIKE '{{ grants_schema_like }}'
+    AND r.rolname LIKE '{{ grants_role_like }}'
+ORDER BY
+    schema_name, role_name

dbt_project/dbt_project.yml

@@ -59,4 +59,7 @@ models:
     combined:
       +tags:
       - combined_stage
-      +schema: combined
+      +schema: combined
+on-run-end:
+  - "{{ grant_devs_access(tag='include', users_role='include_users') }}"
+  - "{{ grant_devs_access(tag='kids_first', users_role='kf_users') }}"

dbt_project/macros/_macros.yml

@@ -185,3 +185,128 @@ macros:
         - delimiter (defaults to ,)
         - default_options (dictionary)
         - descriptor_str_options (dictionary)
+
+  - name: grant_devs_access
+    description: >
+      Shared on-run-end helper that grants a role full access to every schema
+      produced by models matching a given project tag during the current dbt run.
+      The two project-specific wrappers (`grant_inc_devs_access`,
+      `grant_kf_devs_access`) delegate directly to this macro with their
+      fixed `tag` / `users_role` values.
+
+
+      **Behavior:**
+
+      1. Validates that both `tag` and `users_role` are provided; raises a
+         compiler error if either is missing.
+
+      2. Iterates over `results` and collects the schema of every model whose
+         tags contain `tag` — ensuring only schemas belonging to the
+         specified project are touched.
+
+      3. If no matching models are found (e.g. the run contained no models with
+         that tag), logs an INFO message and exits without issuing any SQL.
+
+      4. For each collected schema, checks `has_schema_privilege(..., 'USAGE')`.
+        and `has_schema_privilege(..., 'CREATE')`.
+        Schemas that already have both privileges are skipped with an INFO log.
+
+      5. For schemas that still need granting, issues:
+           - `GRANT USAGE ON SCHEMA` — allows the role to see and interact with the schema.
+         - `GRANT CREATE ON SCHEMA` — allows the role to create objects in the schema.
+           - `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA` — full DML
+             access on every table that currently exists in the schema.
+           - `GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA` — access to all existing
+             sequences (required for auto-increment / serial columns).
+           - `ALTER DEFAULT PRIVILEGES ... GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES`
+             — ensures future tables created in this schema are automatically accessible.
+           - `ALTER DEFAULT PRIVILEGES ... GRANT USAGE, SELECT ON SEQUENCES` — same
+             future coverage for sequences.
+
+
+      6. All actions and skips are logged at `INFO` level.
+
+    arguments:
+      - name: tag
+        type: string
+        description: >
+          The dbt tag used to identify models belonging to the target project.
+          For example, `include` or `kids_first`. Only models whose tags
+          contain this value will have their schemas granted.
+
+      - name: users_role
+        type: string
+        description: >
+          The database role to receive the grants. Must be an existing role
+          in the target database (e.g. `include_users`, `kf_users`).
+
+  - name: grant_inc_devs_access
+    description: >
+      On-run-end wrapper around `grant_devs_access`. Grants `include_users`
+      access to all schemas produced by models tagged `include` in the current
+      run (e.g. `brainpower`, `aadsc` src/int schemas). Delegates entirely to
+      `grant_devs_access(tag='include', users_role='include_users')`.
+
+  - name: grant_kf_devs_access
+    description: >
+      On-run-end wrapper around `grant_devs_access`. Grants `kf_users`
+      access to all schemas produced by models tagged `kids_first` in the
+      current run. Delegates entirely to
+      `grant_devs_access(tag='kids_first', users_role='kf_users')`.
+
+  - name: grant_schema_role_access
+    description: >
+      Standalone utility macro that grants a specified role full access to a target
+      schema. Unlike the on-run-end grant macros (`grant_inc_devs_access`,
+      `grant_kf_devs_access`), this macro is not tied to a dbt run lifecycle and is
+      intended to be called manually via `dbt run-operation` whenever a new role needs
+      access to the `combined`, `access`, or `*_export` schemas.
+
+
+      **Behavior:**
+
+      1. Validates that both `target_schema` and `users_role` are provided; raises a
+         compiler error if either is missing.
+
+      2. Checks whether the role already holds `USAGE` privilege on the schema using
+        `has_schema_privilege`, and also checks `CREATE` on the same schema.
+        If both are already granted, the macro logs a skip
+        message and exits without issuing grant SQL — making it safe to run repeatedly
+         without accumulating duplicate grants.
+
+      3. If access is not yet granted, the macro issues the following statements
+         against the target schema:
+           - `GRANT USAGE ON SCHEMA` — allows the role to see and interact with the schema.
+           - `GRANT CREATE ON SCHEMA` — allows the role to create objects in the schema.
+           - `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA` — full DML
+             access on every table that currently exists in the schema.
+           - `GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA` — access to all existing
+             sequences (required for auto-increment / serial columns).
+           - `ALTER DEFAULT PRIVILEGES ... GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES`
+             — ensures future tables created in this schema are automatically accessible.
+           - `ALTER DEFAULT PRIVILEGES ... GRANT USAGE, SELECT ON SEQUENCES` — same
+             future coverage for sequences.
+
+      4. All actions and skips are logged to the dbt console at `INFO` level.
+
+
+      **Example usage:**
+
+      ```bash
+      dbt run-operation grant_schema_role_access \
+        --args '{"target_schema": "combined", "users_role": "analyst_role"}'
+      ```
+
+    arguments:
+      - name: target_schema
+        type: string
+        description: >
+          The name of the database schema to grant access to. Intended for shared,
+          non-program-specific schemas such as `combined`, `access`, or `fhir_export`.
+
+
+      - name: users_role
+        type: string
+        description: >
+          The name of the database role to receive the grants. This shouldbe 
+          an existing role in the target database.

dbt_project/macros/grants/grant_devs_access.sql

@@ -0,0 +1,77 @@
+{% macro grant_devs_access(tag, users_role) %}
+  {% if execute %}
+    {#
+      Use explicit table privileges instead of GRANT ALL to follow least-privilege.
+      In Postgres, ALL TABLE privileges also include TRIGGER, REFERENCES, and TRUNCATE,
+      which are broader than required for this workflow.
+    #}
+    {% set table_privileges = 'select, insert, update, delete' %}
+    {% set sequence_privileges = 'usage, select' %}
+
+    {% set command_name = flags.WHICH if flags is defined and flags.WHICH is defined else none %}
+    {% if command_name not in ['run', 'build'] %}
+      {% do log('grant_devs_access: skipping for command ' ~ (command_name or 'unknown') ~ '; grants are only applied for run/build', info=True) %}
+      {{ return('') }}
+    {% endif %}
+
+    {% if not tag %}
+      {% do exceptions.raise_compiler_error('grant_devs_access: tag is required') %}
+    {% endif %}
+
+    {% if not users_role %}
+      {% do exceptions.raise_compiler_error('grant_devs_access: users_role is required') %}
+    {% endif %}
+
+    {% set grantee_name = users_role %}
+    {% set grantee = adapter.quote(grantee_name) %}
+    {% set ns = namespace(run_schemas=[]) %}
+
+    {% if results is defined %}
+      {% for res in results %}
+        {% set node_tags = res.node.tags if res.node.tags is defined else [] %}
+        {% if res.node.resource_type == 'model' and tag in node_tags and 'dev_' in (res.node.schema | lower) %}
+          {% do ns.run_schemas.append(res.node.schema) %}
+        {% endif %}
+      {% endfor %}
+    {% endif %}
+
+    {% set run_schemas = ns.run_schemas | unique | list %}
+
+    {% if run_schemas | length == 0 %}
+      {% do log('grant_devs_access: no models with the ' ~ tag ~ ' tag and schemas containing dev_ found in run results; skipping grants', info=True) %}
+    {% endif %}
+
+    {% for schema_name in run_schemas %}
+      {% set quoted_schema = adapter.quote(schema_name) %}
+      {% set schema_granted_sql %}
+          select
+            has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') as has_usage,
+            has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'CREATE') as has_create
+      {% endset %}
+      {% set schema_granted_result = run_query(schema_granted_sql) %}
+        {% set schema_usage_granted = false %}
+        {% set schema_create_granted = false %}
+
+      {% if schema_granted_result is not none and schema_granted_result.rows | length > 0 %}
+          {% set schema_usage_granted = schema_granted_result.rows[0][0] %}
+          {% set schema_create_granted = schema_granted_result.rows[0][1] %}
+      {% endif %}
+
+        {% if not schema_usage_granted or not schema_create_granted %}
+        {% do log('Applying ' ~ grantee_name ~ ' grants on schema ' ~ schema_name, info=True) %}
+
+        {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %}
+          {% do run_query("grant create on schema " ~ quoted_schema ~ " to " ~ grantee) %}
+        {% do run_query("grant " ~ table_privileges ~ " on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %}
+        {% do run_query("grant " ~ sequence_privileges ~ " on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %}
+
+        {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ table_privileges ~ " on tables to " ~ grantee) %}
+        {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ sequence_privileges ~ " on sequences to " ~ grantee) %}
+      {% else %}
+        {% do log('Skipping grants for schema ' ~ schema_name ~ ' (already granted)', info=True) %}
+      {% endif %}
+    {% endfor %}
+  {% endif %}
+
+  {{ return('') }}
+{% endmacro %}

dbt_project/macros/grants/grant_inc_devs_access.sql

@@ -0,0 +1,3 @@
+{% macro grant_inc_devs_access() %}
+  {{ grant_devs_access(tag='include', users_role='include_users') }}
+{% endmacro %}

dbt_project/macros/grants/grant_kf_devs_access.sql

@@ -0,0 +1,3 @@
+{% macro grant_kf_devs_access() %}
+  {{ grant_devs_access(tag='kids_first', users_role='kf_users') }}
+{% endmacro %}

dbt_project/macros/grants/grant_schema_role_access.sql

@@ -0,0 +1,54 @@
+{% macro grant_schema_role_access(target_schema, users_role) %}
+  {% if execute %}
+    {#
+      Use explicit table privileges instead of GRANT ALL to follow least-privilege.
+      In Postgres, ALL TABLE privileges also include TRIGGER, REFERENCES, and TRUNCATE,
+      which are broader than required for this workflow.
+    #}
+    {% set table_privileges = 'select, insert, update, delete' %}
+    {% set sequence_privileges = 'usage, select' %}
+
+    {% if not target_schema %}
+      {% do exceptions.raise_compiler_error('grant_schema_role_access: target_schema is required') %}
+    {% endif %}
+
+    {% if not users_role %}
+      {% do exceptions.raise_compiler_error('grant_schema_role_access: users_role is required') %}
+    {% endif %}
+
+    {% set schema_name = target_schema %}
+    {% set grantee_name = users_role %}
+    {% set quoted_schema = adapter.quote(schema_name) %}
+    {% set grantee = adapter.quote(grantee_name) %}
+
+    {% set schema_granted_sql %}
+      select
+        has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'USAGE') as has_usage,
+        has_schema_privilege('{{ grantee_name }}', '{{ schema_name }}', 'CREATE') as has_create
+    {% endset %}
+    {% set schema_granted_result = run_query(schema_granted_sql) %}
+    {% set schema_usage_granted = false %}
+    {% set schema_create_granted = false %}
+
+    {% if schema_granted_result is not none and schema_granted_result.rows | length > 0 %}
+      {% set schema_usage_granted = schema_granted_result.rows[0][0] %}
+      {% set schema_create_granted = schema_granted_result.rows[0][1] %}
+    {% endif %}
+
+    {% if not schema_usage_granted or not schema_create_granted %}
+      {% do log('Applying ' ~ grantee_name ~ ' grants on schema ' ~ schema_name, info=True) %}
+
+      {% do run_query("grant usage on schema " ~ quoted_schema ~ " to " ~ grantee) %}
+      {% do run_query("grant create on schema " ~ quoted_schema ~ " to " ~ grantee) %}
+      {% do run_query("grant " ~ table_privileges ~ " on all tables in schema " ~ quoted_schema ~ " to " ~ grantee) %}
+      {% do run_query("grant " ~ sequence_privileges ~ " on all sequences in schema " ~ quoted_schema ~ " to " ~ grantee) %}
+
+      {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ table_privileges ~ " on tables to " ~ grantee) %}
+      {% do run_query("alter default privileges in schema " ~ quoted_schema ~ " grant " ~ sequence_privileges ~ " on sequences to " ~ grantee) %}
+    {% else %}
+      {% do log('Skipping grants for schema ' ~ schema_name ~ ' (USAGE and CREATE already granted for role ' ~ grantee_name ~ ')', info=True) %}
+    {% endif %}
+  {% endif %}
+
+  {{ return('') }}
+{% endmacro %}

dbt_project/macros/pipeline/combined_union.sql

@@ -1,21 +1,21 @@
-{% macro combined_stb_relations(table_name, studies_var='combined_studies') %}
-    {% set studies = var(studies_var, []) %}
+{%- macro combined_stb_relations(table_name, studies_var='combined_studies') -%}
+    {%- set studies = var(studies_var, []) -%}
 
-    {% if studies | length == 0 %}
+    {%- if studies | length == 0 -%}
         {{ exceptions.raise_compiler_error("Var '" ~ studies_var ~ "' must contain at least one study prefix.") }}
-    {% endif %}
+    {%- endif -%}
 
-    {% set relations = [] %}
-    {% for study in studies %}
-        {% do relations.append(ref(study ~ '_stb_' ~ table_name)) %}
-    {% endfor %}
+    {%- set relations = [] -%}
+    {%- for study in studies -%}
+        {%- do relations.append(ref(study ~ '_stb_' ~ table_name)) -%}
+    {%- endfor -%}
 
-    {{ return(relations) }}
-{% endmacro %}
+    {{- return(relations) -}}
+{%- endmacro -%}
 
-{% macro combined_union_from_current_model(studies_var='combined_studies') %}
-    {% set table_name = model.name | replace('combined_', '') %}
-    {% set relations = combined_stb_relations(table_name=table_name, studies_var=studies_var) %}
+{%- macro combined_union_from_current_model(studies_var='combined_studies') -%}
+    {%- set table_name = model.name | replace('combined_', '') -%}
+    {%- set relations = combined_stb_relations(table_name=table_name, studies_var=studies_var) -%}
 
-    {{ dbt_utils.union_relations(relations=relations) }}
-{% endmacro %}
+    {{- dbt_utils.union_relations(relations=relations) -}}
+{%- endmacro -%}