🚧 Tweak descriptor view

dbt_project/macros/_macros.yml

@@ -95,22 +95,23 @@ macros:
 
   - name: generate_descriptor_view_sql
     description: |
-      Generates placeholder SQL for a descriptor view based on program and
-      resource exceptions.
-      Used by descriptor-view models that need a compile-safe SQL stub.
+      Generates a descriptor view by collecting unique descriptors from
+      available int tables and returning only values missing from
+      inc_brainpower_raw.global_ids for the same fhirResourceType and study.
+
+      Source relation resolution:
+        - Each source is resolved as {table_prefix}{relation_name}
+        - Missing relations are skipped automatically
 
       Quickstart:
         macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md
 
       Required args:
-        - program_id
-        - dewrangle_study_id
-      Optional args:
-        - inclusion_resources (list)
-        - exclusion_resources (list)
+        - table_prefix
+        - study_global_id
 
       Used from model SQL, for example:
-        generate_descriptor_view_sql(program_id='inc', dewrangle_study_id='brainpower', exclusion_resources=[])
+        generate_descriptor_view_sql(table_prefix='inc_brainpower_int_', study_global_id='brainpower')
 
   - name: generate_schema_name
     description: |
@@ -135,10 +136,10 @@ macros:
         - base_source
       Optional args:
         - gid_lookup
-        - descriptor_sources (list)
+        - descriptor_sources (list of resource names)
 
       Used from model SQL, for example:
-        generate_stb_sql(base_source='my_base_model', gid_lookup='my_gid_lookup', descriptor_sources=[{'Patient': {} }])
+        generate_stb_sql(base_source='my_base_model', gid_lookup='my_gid_lookup', descriptor_sources=['Patient'])
 
   - name: normalize_descriptors
     description: |

dbt_project/macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md

@@ -1,41 +1,34 @@
 # generate_descriptor_view_sql Quickstart
 
-Use this macro to emit a compile-safe descriptor-view SQL placeholder.
+Use this macro to build a descriptor view from available int descriptor tables.
+It returns unique descriptors that are missing from `inc_brainpower_raw.global_ids`
+for the same `fhirResourceType` and study.
 
 ## Basic Usage
 
 ```jinja
 {{ config(materialized='table') }}
 
 {{ generate_descriptor_view_sql(
-    program_id='inc',
-    dewrangle_study_id='brainpower'
+    table_prefix='inc_brainpower_int_',
+    study_global_id='brainpower'
 ) }}
 ```
 
-## Exclude Specific Resource Models
+## Arguments
 
-```jinja
-{{ generate_descriptor_view_sql(
-    program_id='inc',
-    dewrangle_study_id='brainpower',
-    exclusion_resources=['AccessPolicy']
-) }}
-```
+- table_prefix: prefix used to resolve descriptor source relations
+- study_global_id: value matched to `global_ids.studyId`, and emitted as `studyGlobalId`
 
-## Include Only Specific Resource Models
+## Relation Resolution
 
-```jinja
-{{ generate_descriptor_view_sql(
-    program_id='inc',
-    dewrangle_study_id='brainpower',
-    inclusion_resources=['Demographics']
-) }}
-```
+The macro attempts each source as:
 
-## Valid Values
+- `{table_prefix}accesspolicy`
+- `{table_prefix}demographics`
+- `{table_prefix}family`
 
-- program_id: `inc` or `kf`
-- inclusion_resources/exclusion_resources items: `AccessPolicy`, `Demographics`
+If a relation does not exist, it is skipped automatically.
 
-Invalid values raise compile errors.
+The macro automatically uses descriptor-source int models that exist and are
+built; missing sources are skipped.

dbt_project/macros/pipeline/docs/generate_stb_sql_quickstart.md

@@ -9,7 +9,7 @@ Use this macro to join descriptor-based source rows to global IDs (GIDs).
 ## Optional Input
 
 - gid_lookup: model name or relation for GID table
-- descriptor_sources: list describing which resource mappings to include
+- descriptor_sources: list of resource names to include
 
 ## Basic Usage
 
@@ -18,8 +18,8 @@ Use this macro to join descriptor-based source rows to global IDs (GIDs).
     gid_lookup='inc_brainpower_int_global_ids',
     base_source='inc_brainpower_int_accesspolicy',
     descriptor_sources=[
-      {'AccessPolicy': {}},
-      {'Patient': {}}
+      'AccessPolicy',
+      'Patient'
     ]
 ) }}
 ```

dbt_project/macros/pipeline/generate_descriptor_view_sql.sql

@@ -1,60 +1,51 @@
-{%- macro generate_descriptor_view_sql(program_id='', dewrangle_study_id='', inclusion_resources=[], exclusion_resources=[]) -%}
+{%- macro generate_descriptor_view_sql(table_prefix='', study_global_id='') -%}
 
-{%- set accepted_models = ['AccessPolicy', 'Demographics'] -%}
-{%- set accepted_program_ids = ['inc', 'kf'] -%}
 {%- set descriptor_sources = [
-    {'resource_model': 'AccessPolicy', 'relation_name': 'inc_brainpower_int_accesspolicy', 'descriptor_column': 'access_policy_descriptor'},
-    {'resource_model': 'AccessPolicy', 'relation_name': 'inc_brainpower_int_demographics', 'descriptor_column': 'access_policy_descriptor'},
-    {'resource_model': 'Patient', 'relation_name': 'inc_brainpower_int_demographics', 'descriptor_column': 'subject_descriptor'},
-    {'resource_model': 'ResearchStudy', 'relation_name': 'inc_brainpower_int_demographics', 'descriptor_column': 'study_descriptor'}
+    {'resource_model': 'AccessPolicy', 'relation_name': 'accesspolicy', 'descriptor_column': 'access_policy_descriptor'},
+    {'resource_model': 'AccessPolicy', 'relation_name': 'demographics', 'descriptor_column': 'access_policy_descriptor'},
+    {'resource_model': 'Patient', 'relation_name': 'demographics', 'descriptor_column': 'subject_descriptor'},
+    {'resource_model': 'ResearchStudy', 'relation_name': 'demographics', 'descriptor_column': 'study_descriptor'},
+    {'resource_model': 'Family', 'relation_name': 'family', 'descriptor_column': 'study_descriptor'}
 ] -%}
 
-{%- if program_id not in accepted_program_ids -%}
-    {%- do exceptions.raise_compiler_error("Invalid program_id: " ~ program_id ~ ". Accepted program IDs are: " ~ accepted_program_ids | join(", ")) -%}
-{%- endif -%}
-
-{%- for model in inclusion_resources -%}
-    {%- if model not in accepted_models -%}
-        {%- do exceptions.raise_compiler_error("Invalid model type: " ~ model ~ ". Accepted models are: " ~ accepted_models | join(", ")) -%}
-    {%- endif -%}
-{%- endfor -%}
-
-{%- for model in exclusion_resources -%}
-    {%- if model not in accepted_models -%}
-        {%- do exceptions.raise_compiler_error("Invalid model type: " ~ model ~ ". Accepted models are: " ~ accepted_models | join(", ")) -%}
-    {%- endif -%}
-{%- endfor -%}
-
-{%- set curated_models = [] -%}
-{%- if inclusion_resources | length > 0 -%}
-    {%- for model in inclusion_resources -%}
-        {%- do curated_models.append(model) -%}
-    {%- endfor -%}
-{%- else -%}
-    {%- for model in accepted_models -%}
-        {%- do curated_models.append(model) -%}
-    {%- endfor -%}
-{%- endif -%}
 
 {%- set curated_sources = [] -%}
 {%- for source_row in descriptor_sources -%}
-    {%- if source_row['resource_model'] == 'AccessPolicy' and 'AccessPolicy' in curated_models and 'AccessPolicy' not in exclusion_resources -%}
-        {%- do curated_sources.append(source_row) -%}
-    {%- elif source_row['resource_model'] in ['Patient', 'ResearchStudy'] and 'Demographics' in curated_models and 'Demographics' not in exclusion_resources -%}
-        {%- do curated_sources.append(source_row) -%}
+    {%- set source_identifier = table_prefix ~ source_row['relation_name'] -%}
+    {%- if execute -%}
+        {%- set descriptor_relation = adapter.get_relation(
+            database=this.database,
+            schema=this.schema,
+            identifier=source_identifier
+        ) -%}
+        {%- if descriptor_relation is not none -%}
+            {%- do curated_sources.append({'resource_model': source_row['resource_model'], 'relation': descriptor_relation, 'descriptor_column': source_row['descriptor_column']}) -%}
+        {%- endif -%}
+    {%- else -%}
+        {%- set descriptor_relation = api.Relation.create(
+            database=this.database,
+            schema=this.schema,
+            identifier=source_identifier
+        ) -%}
+        {%- do curated_sources.append({'resource_model': source_row['resource_model'], 'relation': descriptor_relation, 'descriptor_column': source_row['descriptor_column']}) -%}
     {%- endif -%}
 {%- endfor -%}
 
 {%- if curated_sources | length == 0 -%}
-    {{ exceptions.raise_compiler_error("No descriptor sources selected. Check inclusion_resources/exclusion_resources values.") }}
-{%- endif -%}
+select
+    '{{ study_global_id }}'::text as "studyGlobalId",
+    cast(null as text) as "fhirResourceType",
+    cast(null as text) as descriptor,
+    cast(null as text) as "descriptorState"
+where 1 = 0
+{%- else -%}
 
 with int_descriptors as (
 {%- for source_row in curated_sources %}
     select
         '{{ source_row['resource_model'] }}'::text as resource_model,
         {{ source_row['descriptor_column'] }}::text as descriptor
-    from {{ ref(source_row['relation_name']) }}
+    from {{ source_row['relation'] }}
 {% if not loop.last %}
     union all
 
@@ -75,19 +66,20 @@ existing_global_ids as (
         "fhirResourceType"::text as resource_model,
         descriptor::text as descriptor
     from {{ source('brainpower', 'global_ids') }}
-    where "studyId" = '{{ dewrangle_study_id }}'
+    where "studyId" = '{{ study_global_id }}'
 )
 
 select
-    '{{ program_id }}'::text as program_id,
-    '{{ dewrangle_study_id }}'::text as study_id,
-    u.resource_model,
-    u.descriptor
+    '{{ study_global_id }}'::text as "studyGlobalId",
+    u.resource_model as "fhirResourceType",
+    u.descriptor as descriptor,
+    'ACTIVE'::text as "descriptorState"
 from unique_descriptors as u
 left join existing_global_ids as g
   on u.resource_model = g.resource_model
  and u.descriptor = g.descriptor
 where g.descriptor is null
 order by u.resource_model, u.descriptor
+{%- endif -%}
 
 {%- endmacro -%}

dbt_project/macros/pipeline/generate_stb_sql.sql

@@ -20,25 +20,25 @@
 
 {%- set descriptor_mappings = [] -%}
 {%- for descriptor_source in descriptor_sources -%}
-  {%- for resource_name, _ in descriptor_source.items() -%}
-    {%- set resource_key = resource_name | lower -%}
-    {%- if resource_key in resource_config -%}
-      {%- set entry = resource_config[resource_key].copy() -%}
-      {%- do entry.update({'resource_key': resource_key}) -%}
-      {%- do descriptor_mappings.append(entry) -%}
-    {%- endif -%}
-  {%- endfor -%}
+  {%- set resource_key = descriptor_source | lower -%}
+  {%- if resource_key in resource_config -%}
+    {%- set entry = resource_config[resource_key].copy() -%}
+    {%- do entry.update({'resource_key': resource_key}) -%}
+    {%- do descriptor_mappings.append(entry) -%}
+  {%- else -%}
+    {{ exceptions.raise_compiler_error("Unsupported descriptor source: " ~ descriptor_source ~ ". Supported values are: AccessPolicy, ActivityDefinition, Patient") }}
+  {%- endif -%}
 {%- endfor -%}
 
 with
 {%- for mapping in descriptor_mappings %}
   filtered_{{ mapping['resource_key'] }}_gids as (
     select
-      globalId as {{ mapping['gid_column'] }},
+      "globalId" as {{ mapping['gid_column'] }},
       descriptor as {{ mapping['descriptor_column'] }}
     from {{ gid_lookup }}
-    where fhirResourceType = '{{ mapping['resource'] }}'
-      and studyId = 'brainpower'
+    where "fhirResourceType" = '{{ mapping['resource'] }}'
+      and "studyId" = 'brainpower'
   ){% if not loop.last %},{% endif %}
 {%- endfor %}
 

dbt_project/models/include/brainpower/int/inc_brainpower_int_descriptors.sql

@@ -1,5 +1,5 @@
 {{ config(materialized='view') }}
 
 {{ generate_descriptor_view_sql(
-program_id='inc', dewrangle_study_id='brainpower', exclusion_resources=[]
+table_prefix='inc_brainpower_int_', study_global_id='brainpower'
 ) }}

dbt_project/models/include/brainpower/stb/inc_brainpower_stb_accesspolicy.sql

@@ -4,6 +4,6 @@
     gid_lookup=source('brainpower', 'global_ids'),
     base_source=ref('inc_brainpower_int_accesspolicy'),
     descriptor_sources=[
-        {'AccessPolicy': ref('inc_brainpower_src_datasets')}
+        'AccessPolicy'
     ]
 ) }}

dbt_project/models/include/brainpower/stb/inc_brainpower_stb_subject.sql

@@ -1,13 +1,9 @@
 {{ config(materialized='table') }}
 
-select
-  null::text as subject_id,
-  null::text as subject_type,
-  null::text as organism_type,
-  null::text as access_policy_id,
-  null::text as study_id,
-  null::text as external_id
-from {{ ref('inc_brainpower_src_inc_kf_access_enums_20260508') }}
-where enumeration_code = 'participant'
-
-
+{{ generate_stb_sql(
+    gid_lookup=source('brainpower', 'global_ids'),
+    base_source=ref('inc_brainpower_int_subject'),
+    descriptor_sources=[
+        'Patient'
+    ]
+) }}