From 4da4797042211c24a866524a43cbcbae00869d62 Mon Sep 17 00:00:00 2001 From: brendagutman Date: Fri, 24 Jul 2026 13:45:34 -0500 Subject: [PATCH 1/3] Modify descriptor view --- dbt_project/macros/_macros.yml | 19 +++-- ...generate_descriptor_view_sql_quickstart.md | 39 ++++----- .../pipeline/generate_descriptor_view_sql.sql | 82 +++++++++---------- .../int/inc_brainpower_int_descriptors.sql | 2 +- 4 files changed, 64 insertions(+), 78 deletions(-) diff --git a/dbt_project/macros/_macros.yml b/dbt_project/macros/_macros.yml index e8a7ad4..b285da6 100644 --- a/dbt_project/macros/_macros.yml +++ b/dbt_project/macros/_macros.yml @@ -95,22 +95,23 @@ macros: - name: generate_descriptor_view_sql description: | - Generates placeholder SQL for a descriptor view based on program and - resource exceptions. - Used by descriptor-view models that need a compile-safe SQL stub. + Generates a descriptor view by collecting unique descriptors from + available int tables and returning only values missing from + inc_brainpower_raw.global_ids for the same fhirResourceType and study. + + Source relation resolution: + - Each source is resolved as {table_prefix}{relation_name} + - Missing relations are skipped automatically Quickstart: macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md Required args: - - program_id - - dewrangle_study_id - Optional args: - - inclusion_resources (list) - - exclusion_resources (list) + - table_prefix + - study_global_id Used from model SQL, for example: - generate_descriptor_view_sql(program_id='inc', dewrangle_study_id='brainpower', exclusion_resources=[]) + generate_descriptor_view_sql(table_prefix='inc_brainpower_int_', study_global_id='brainpower') - name: generate_schema_name description: | diff --git a/dbt_project/macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md b/dbt_project/macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md index f36cff9..2863e30 100644 --- a/dbt_project/macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md +++ b/dbt_project/macros/pipeline/docs/generate_descriptor_view_sql_quickstart.md @@ -1,6 +1,8 @@ # generate_descriptor_view_sql Quickstart -Use this macro to emit a compile-safe descriptor-view SQL placeholder. +Use this macro to build a descriptor view from available int descriptor tables. +It returns unique descriptors that are missing from `inc_brainpower_raw.global_ids` +for the same `fhirResourceType` and study. ## Basic Usage @@ -8,34 +10,25 @@ Use this macro to emit a compile-safe descriptor-view SQL placeholder. {{ config(materialized='table') }} {{ generate_descriptor_view_sql( - program_id='inc', - dewrangle_study_id='brainpower' + table_prefix='inc_brainpower_int_', + study_global_id='brainpower' ) }} ``` -## Exclude Specific Resource Models +## Arguments -```jinja -{{ generate_descriptor_view_sql( - program_id='inc', - dewrangle_study_id='brainpower', - exclusion_resources=['AccessPolicy'] -) }} -``` +- table_prefix: prefix used to resolve descriptor source relations +- study_global_id: value matched to `global_ids.studyId`, and emitted as `studyGlobalId` -## Include Only Specific Resource Models +## Relation Resolution -```jinja -{{ generate_descriptor_view_sql( - program_id='inc', - dewrangle_study_id='brainpower', - inclusion_resources=['Demographics'] -) }} -``` +The macro attempts each source as: -## Valid Values +- `{table_prefix}accesspolicy` +- `{table_prefix}demographics` +- `{table_prefix}family` -- program_id: `inc` or `kf` -- inclusion_resources/exclusion_resources items: `AccessPolicy`, `Demographics` +If a relation does not exist, it is skipped automatically. -Invalid values raise compile errors. +The macro automatically uses descriptor-source int models that exist and are +built; missing sources are skipped. diff --git a/dbt_project/macros/pipeline/generate_descriptor_view_sql.sql b/dbt_project/macros/pipeline/generate_descriptor_view_sql.sql index 66fc94b..dbf6efc 100644 --- a/dbt_project/macros/pipeline/generate_descriptor_view_sql.sql +++ b/dbt_project/macros/pipeline/generate_descriptor_view_sql.sql @@ -1,60 +1,51 @@ -{%- macro generate_descriptor_view_sql(program_id='', dewrangle_study_id='', inclusion_resources=[], exclusion_resources=[]) -%} +{%- macro generate_descriptor_view_sql(table_prefix='', study_global_id='') -%} -{%- set accepted_models = ['AccessPolicy', 'Demographics'] -%} -{%- set accepted_program_ids = ['inc', 'kf'] -%} {%- set descriptor_sources = [ - {'resource_model': 'AccessPolicy', 'relation_name': 'inc_brainpower_int_accesspolicy', 'descriptor_column': 'access_policy_descriptor'}, - {'resource_model': 'AccessPolicy', 'relation_name': 'inc_brainpower_int_demographics', 'descriptor_column': 'access_policy_descriptor'}, - {'resource_model': 'Patient', 'relation_name': 'inc_brainpower_int_demographics', 'descriptor_column': 'subject_descriptor'}, - {'resource_model': 'ResearchStudy', 'relation_name': 'inc_brainpower_int_demographics', 'descriptor_column': 'study_descriptor'} + {'resource_model': 'AccessPolicy', 'relation_name': 'accesspolicy', 'descriptor_column': 'access_policy_descriptor'}, + {'resource_model': 'AccessPolicy', 'relation_name': 'demographics', 'descriptor_column': 'access_policy_descriptor'}, + {'resource_model': 'Patient', 'relation_name': 'demographics', 'descriptor_column': 'subject_descriptor'}, + {'resource_model': 'ResearchStudy', 'relation_name': 'demographics', 'descriptor_column': 'study_descriptor'}, + {'resource_model': 'Family', 'relation_name': 'family', 'descriptor_column': 'study_descriptor'} ] -%} -{%- if program_id not in accepted_program_ids -%} - {%- do exceptions.raise_compiler_error("Invalid program_id: " ~ program_id ~ ". Accepted program IDs are: " ~ accepted_program_ids | join(", ")) -%} -{%- endif -%} - -{%- for model in inclusion_resources -%} - {%- if model not in accepted_models -%} - {%- do exceptions.raise_compiler_error("Invalid model type: " ~ model ~ ". Accepted models are: " ~ accepted_models | join(", ")) -%} - {%- endif -%} -{%- endfor -%} - -{%- for model in exclusion_resources -%} - {%- if model not in accepted_models -%} - {%- do exceptions.raise_compiler_error("Invalid model type: " ~ model ~ ". Accepted models are: " ~ accepted_models | join(", ")) -%} - {%- endif -%} -{%- endfor -%} - -{%- set curated_models = [] -%} -{%- if inclusion_resources | length > 0 -%} - {%- for model in inclusion_resources -%} - {%- do curated_models.append(model) -%} - {%- endfor -%} -{%- else -%} - {%- for model in accepted_models -%} - {%- do curated_models.append(model) -%} - {%- endfor -%} -{%- endif -%} {%- set curated_sources = [] -%} {%- for source_row in descriptor_sources -%} - {%- if source_row['resource_model'] == 'AccessPolicy' and 'AccessPolicy' in curated_models and 'AccessPolicy' not in exclusion_resources -%} - {%- do curated_sources.append(source_row) -%} - {%- elif source_row['resource_model'] in ['Patient', 'ResearchStudy'] and 'Demographics' in curated_models and 'Demographics' not in exclusion_resources -%} - {%- do curated_sources.append(source_row) -%} + {%- set source_identifier = table_prefix ~ source_row['relation_name'] -%} + {%- if execute -%} + {%- set descriptor_relation = adapter.get_relation( + database=this.database, + schema=this.schema, + identifier=source_identifier + ) -%} + {%- if descriptor_relation is not none -%} + {%- do curated_sources.append({'resource_model': source_row['resource_model'], 'relation': descriptor_relation, 'descriptor_column': source_row['descriptor_column']}) -%} + {%- endif -%} + {%- else -%} + {%- set descriptor_relation = api.Relation.create( + database=this.database, + schema=this.schema, + identifier=source_identifier + ) -%} + {%- do curated_sources.append({'resource_model': source_row['resource_model'], 'relation': descriptor_relation, 'descriptor_column': source_row['descriptor_column']}) -%} {%- endif -%} {%- endfor -%} {%- if curated_sources | length == 0 -%} - {{ exceptions.raise_compiler_error("No descriptor sources selected. Check inclusion_resources/exclusion_resources values.") }} -{%- endif -%} +select + '{{ study_global_id }}'::text as "studyGlobalId", + cast(null as text) as "fhirResourceType", + cast(null as text) as descriptor, + cast(null as text) as "descriptorState" +where 1 = 0 +{%- else -%} with int_descriptors as ( {%- for source_row in curated_sources %} select '{{ source_row['resource_model'] }}'::text as resource_model, {{ source_row['descriptor_column'] }}::text as descriptor - from {{ ref(source_row['relation_name']) }} + from {{ source_row['relation'] }} {% if not loop.last %} union all @@ -75,19 +66,20 @@ existing_global_ids as ( "fhirResourceType"::text as resource_model, descriptor::text as descriptor from {{ source('brainpower', 'global_ids') }} - where "studyId" = '{{ dewrangle_study_id }}' + where "studyId" = '{{ study_global_id }}' ) select - '{{ program_id }}'::text as program_id, - '{{ dewrangle_study_id }}'::text as study_id, - u.resource_model, - u.descriptor + '{{ study_global_id }}'::text as "studyGlobalId", + u.resource_model as "fhirResourceType", + u.descriptor as descriptor, + 'ACTIVE'::text as "descriptorState" from unique_descriptors as u left join existing_global_ids as g on u.resource_model = g.resource_model and u.descriptor = g.descriptor where g.descriptor is null order by u.resource_model, u.descriptor +{%- endif -%} {%- endmacro -%} \ No newline at end of file diff --git a/dbt_project/models/include/brainpower/int/inc_brainpower_int_descriptors.sql b/dbt_project/models/include/brainpower/int/inc_brainpower_int_descriptors.sql index f8ff51b..05e2dfb 100644 --- a/dbt_project/models/include/brainpower/int/inc_brainpower_int_descriptors.sql +++ b/dbt_project/models/include/brainpower/int/inc_brainpower_int_descriptors.sql @@ -1,5 +1,5 @@ {{ config(materialized='view') }} {{ generate_descriptor_view_sql( -program_id='inc', dewrangle_study_id='brainpower', exclusion_resources=[] +table_prefix='inc_brainpower_int_', study_global_id='brainpower' ) }} \ No newline at end of file From fd21c07948f3a20e2ee9080001481c1b4eada663 Mon Sep 17 00:00:00 2001 From: brendagutman Date: Fri, 24 Jul 2026 13:54:25 -0500 Subject: [PATCH 2/3] Modify stb sql macro to handle casing --- dbt_project/macros/pipeline/generate_stb_sql.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt_project/macros/pipeline/generate_stb_sql.sql b/dbt_project/macros/pipeline/generate_stb_sql.sql index aa554be..31b0ea0 100644 --- a/dbt_project/macros/pipeline/generate_stb_sql.sql +++ b/dbt_project/macros/pipeline/generate_stb_sql.sql @@ -34,11 +34,11 @@ with {%- for mapping in descriptor_mappings %} filtered_{{ mapping['resource_key'] }}_gids as ( select - globalId as {{ mapping['gid_column'] }}, + "globalId" as {{ mapping['gid_column'] }}, descriptor as {{ mapping['descriptor_column'] }} from {{ gid_lookup }} - where fhirResourceType = '{{ mapping['resource'] }}' - and studyId = 'brainpower' + where "fhirResourceType" = '{{ mapping['resource'] }}' + and "studyId" = 'brainpower' ){% if not loop.last %},{% endif %} {%- endfor %} From b7e42c5b4602ce6a6d791ad9953820f40a44c0e5 Mon Sep 17 00:00:00 2001 From: brendagutman Date: Fri, 24 Jul 2026 14:09:55 -0500 Subject: [PATCH 3/3] Modify stb sql simplify --- dbt_project/macros/_macros.yml | 4 ++-- .../docs/generate_stb_sql_quickstart.md | 6 +++--- .../macros/pipeline/generate_stb_sql.sql | 16 ++++++++-------- .../stb/inc_brainpower_stb_accesspolicy.sql | 2 +- .../stb/inc_brainpower_stb_subject.sql | 18 +++++++----------- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/dbt_project/macros/_macros.yml b/dbt_project/macros/_macros.yml index b285da6..dceddef 100644 --- a/dbt_project/macros/_macros.yml +++ b/dbt_project/macros/_macros.yml @@ -136,10 +136,10 @@ macros: - base_source Optional args: - gid_lookup - - descriptor_sources (list) + - descriptor_sources (list of resource names) Used from model SQL, for example: - generate_stb_sql(base_source='my_base_model', gid_lookup='my_gid_lookup', descriptor_sources=[{'Patient': {} }]) + generate_stb_sql(base_source='my_base_model', gid_lookup='my_gid_lookup', descriptor_sources=['Patient']) - name: normalize_descriptors description: | diff --git a/dbt_project/macros/pipeline/docs/generate_stb_sql_quickstart.md b/dbt_project/macros/pipeline/docs/generate_stb_sql_quickstart.md index 64b0f4c..559c2ac 100644 --- a/dbt_project/macros/pipeline/docs/generate_stb_sql_quickstart.md +++ b/dbt_project/macros/pipeline/docs/generate_stb_sql_quickstart.md @@ -9,7 +9,7 @@ Use this macro to join descriptor-based source rows to global IDs (GIDs). ## Optional Input - gid_lookup: model name or relation for GID table -- descriptor_sources: list describing which resource mappings to include +- descriptor_sources: list of resource names to include ## Basic Usage @@ -18,8 +18,8 @@ Use this macro to join descriptor-based source rows to global IDs (GIDs). gid_lookup='inc_brainpower_int_global_ids', base_source='inc_brainpower_int_accesspolicy', descriptor_sources=[ - {'AccessPolicy': {}}, - {'Patient': {}} + 'AccessPolicy', + 'Patient' ] ) }} ``` diff --git a/dbt_project/macros/pipeline/generate_stb_sql.sql b/dbt_project/macros/pipeline/generate_stb_sql.sql index 31b0ea0..35b113a 100644 --- a/dbt_project/macros/pipeline/generate_stb_sql.sql +++ b/dbt_project/macros/pipeline/generate_stb_sql.sql @@ -20,14 +20,14 @@ {%- set descriptor_mappings = [] -%} {%- for descriptor_source in descriptor_sources -%} - {%- for resource_name, _ in descriptor_source.items() -%} - {%- set resource_key = resource_name | lower -%} - {%- if resource_key in resource_config -%} - {%- set entry = resource_config[resource_key].copy() -%} - {%- do entry.update({'resource_key': resource_key}) -%} - {%- do descriptor_mappings.append(entry) -%} - {%- endif -%} - {%- endfor -%} + {%- set resource_key = descriptor_source | lower -%} + {%- if resource_key in resource_config -%} + {%- set entry = resource_config[resource_key].copy() -%} + {%- do entry.update({'resource_key': resource_key}) -%} + {%- do descriptor_mappings.append(entry) -%} + {%- else -%} + {{ exceptions.raise_compiler_error("Unsupported descriptor source: " ~ descriptor_source ~ ". Supported values are: AccessPolicy, ActivityDefinition, Patient") }} + {%- endif -%} {%- endfor -%} with diff --git a/dbt_project/models/include/brainpower/stb/inc_brainpower_stb_accesspolicy.sql b/dbt_project/models/include/brainpower/stb/inc_brainpower_stb_accesspolicy.sql index 03d1436..c9666c7 100644 --- a/dbt_project/models/include/brainpower/stb/inc_brainpower_stb_accesspolicy.sql +++ b/dbt_project/models/include/brainpower/stb/inc_brainpower_stb_accesspolicy.sql @@ -4,6 +4,6 @@ gid_lookup=source('brainpower', 'global_ids'), base_source=ref('inc_brainpower_int_accesspolicy'), descriptor_sources=[ - {'AccessPolicy': ref('inc_brainpower_src_datasets')} + 'AccessPolicy' ] ) }} \ No newline at end of file diff --git a/dbt_project/models/include/brainpower/stb/inc_brainpower_stb_subject.sql b/dbt_project/models/include/brainpower/stb/inc_brainpower_stb_subject.sql index 2e84744..b9607e0 100644 --- a/dbt_project/models/include/brainpower/stb/inc_brainpower_stb_subject.sql +++ b/dbt_project/models/include/brainpower/stb/inc_brainpower_stb_subject.sql @@ -1,13 +1,9 @@ {{ config(materialized='table') }} -select - null::text as subject_id, - null::text as subject_type, - null::text as organism_type, - null::text as access_policy_id, - null::text as study_id, - null::text as external_id -from {{ ref('inc_brainpower_src_inc_kf_access_enums_20260508') }} -where enumeration_code = 'participant' - - \ No newline at end of file +{{ generate_stb_sql( + gid_lookup=source('brainpower', 'global_ids'), + base_source=ref('inc_brainpower_int_subject'), + descriptor_sources=[ + 'Patient' + ] +) }} \ No newline at end of file