Add sensitive-data-flow check

.github/workflows/build.yml

@@ -7,6 +7,7 @@ on:
     types:
       - opened
       - synchronize
+      - ready_for_review
   workflow_run:
     workflows: ["Fork PR Gate"]
     types: [completed]
@@ -303,6 +304,128 @@ jobs:
           PATH=".venv/bin:$PATH" script/bqetl format --check \
           $(git ls-tree -d HEAD --name-only)
 
+  flag-sensitive-flows:
+    name: Flag sensitive data flows
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write  # to request @mozilla/dataplatform-wg review
+    environment: *build-env
+    needs: [build, decide-runs]
+    if: needs.decide-runs.outputs.validate-sql == 'true'
+    steps:
+      - *checkout-with-history
+      - *setup-python
+      - *restore-venv
+      # Resolve PR number / base SHA / draft across event types. On the
+      # workflow_run path (fork PRs, post gate-approval) github.event.pull_request
+      # is null, so resolve from the workflow_run payload / API instead — mirrors
+      # resolve-pr-base and Determine context elsewhere in this workflow.
+      - id: pr-context
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.event.pull_request.head.sha || github.sha }}
+          PR_FROM_WR: ${{ github.event.workflow_run.pull_requests[0].number }}
+          EVENT_PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          EVENT_DRAFT: ${{ github.event.pull_request.draft }}
+        run: |
+          set -euo pipefail
+          pr_number=""; base_sha=""; draft=""
+          if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+            pr_number="$EVENT_PR_NUMBER"
+            base_sha="$EVENT_BASE_SHA"
+            draft="$EVENT_DRAFT"
+          elif [[ "$GITHUB_EVENT_NAME" == "workflow_run" ]]; then
+            pr_number="$PR_FROM_WR"
+            if [[ -z "$pr_number" || "$pr_number" == "null" ]]; then
+              pr_number=$(gh api "/repos/${REPO}/commits/${HEAD_SHA}/pulls" \
+                --jq '.[0].number' 2>/dev/null || echo "")
+            fi
+            if [[ -n "$pr_number" && "$pr_number" != "null" ]]; then
+              base_sha=$(gh api "/repos/${REPO}/pulls/${pr_number}" \
+                --jq '.base.sha' 2>/dev/null || echo "")
+              draft=$(gh api "/repos/${REPO}/pulls/${pr_number}" \
+                --jq '.draft' 2>/dev/null || echo "")
+            fi
+          fi
+          echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT"
+          echo "base_sha=${base_sha}" >> "$GITHUB_OUTPUT"
+          echo "draft=${draft}" >> "$GITHUB_OUTPUT"
+      - id: changed-queries
+        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96  # v47.0.6
+        with:
+          files: sql/**/query.sql
+          base_sha: ${{ steps.pr-context.outputs.base_sha || github.event.pull_request.base.sha || github.event.before }}
+      # A changed query reads sensitive (restricted/workgroup-gated) data and
+      # writes to a more broadly readable destination that no team owns. This is
+      # advisory: rather than block or make the author edit CODEOWNERS, it just
+      # requests Data Platform review and posts a comment. Flows on paths already
+      # owned by a team are left to that team's normal review.
+      - name: Flag sensitive-data flows widening read access
+        if: steps.changed-queries.outputs.any_changed == 'true'
+        env:
+          CHANGED_QUERIES: ${{ steps.changed-queries.outputs.all_changed_files }}
+          GH_TOKEN: ${{ github.token }}
+          PR: ${{ steps.pr-context.outputs.pr_number }}
+          REPO: ${{ github.repository }}
+          DRAFT: ${{ steps.pr-context.outputs.draft }}
+        run: |
+          set +e
+          advisory=$(PATH=".venv/bin:$PATH" \
+            script/bqetl data_governance sensitivity $CHANGED_QUERIES)
+          code=$?
+          set -e
+          echo "$advisory"
+          # Detection always runs (visible above), but hold off on requesting
+          # review / commenting while the PR is a draft — the ready_for_review
+          # trigger re-runs this and posts once it's marked ready.
+          if [ "$code" -eq 2 ] && [ -n "$PR" ] && [ "$DRAFT" != "true" ]; then
+            # The @mozilla/dataplatform-wg mention in the comment below is the
+            # reliable notification. Also try to add the team to the Reviewers
+            # box, but this is best-effort: the repo-scoped Actions GITHUB_TOKEN
+            # can't resolve an org team as a reviewer (needs read:org, which the
+            # permissions: block can't grant), so it 422s — a PAT/App token
+            # secret with read:org would make it work.
+            gh api --method POST \
+              "repos/$REPO/pulls/$PR/requested_reviewers" \
+              -f "team_reviewers[]=dataplatform-wg" \
+              || echo "Note: could not add the team to the Reviewers box with the" \
+                      "Actions token; the @mention in the comment notifies them."
+            # Upsert a single advisory comment (marker-keyed) so re-runs on each
+            # push update it in place instead of piling up duplicates.
+            marker='<!-- sensitive-data-flow -->'
+            body="$(mktemp)"
+            {
+              echo "$marker"
+              echo "⚠️ **Sensitive-data flow detected** — cc @mozilla/dataplatform-wg, please review."
+              echo
+              echo "A changed query reads restricted / workgroup-gated data and writes it to a more broadly readable destination that no team owns:"
+              echo
+              echo '```'
+              echo "$advisory"
+              echo '```'
+              echo
+              echo "This is advisory and does not block merge."
+              echo "A Data Platform reviewer should confirm the widened access is intended, or narrow the destination's \`workgroup_access\`."
+            } > "$body"
+            existing=$(gh api "repos/$REPO/issues/$PR/comments" \
+              --jq "[.[] | select(.body | contains(\"$marker\"))][0].id" 2>/dev/null || echo "")
+            if [ -n "$existing" ] && [ "$existing" != "null" ]; then
+              gh api --method PATCH "repos/$REPO/issues/comments/$existing" \
+                -F "body=@$body" >/dev/null || echo "Could not update existing comment"
+            else
+              gh pr comment "$PR" -R "$REPO" --body-file "$body" \
+                || echo "Could not post comment (fork PR token is read-only?)"
+            fi
+          elif [ "$code" -eq 2 ] && [ "$DRAFT" = "true" ]; then
+            echo "::notice::Sensitive-data flow(s) detected; PR is a draft, deferring @mozilla/dataplatform-wg review until it's marked ready."
+          elif [ "$code" -ne 0 ] && [ "$code" -ne 2 ]; then
+            echo "::warning::sensitivity check errored (exit $code); skipping"
+          fi
+          exit $code
+
   test-bqetl:
     name: Test bqetl
     runs-on: ubuntu-latest

bigquery_etl/data_governance/cli.py

@@ -1,11 +1,14 @@
 """bigquery-etl CLI data_governance command."""
 
+import json
+import sys
 from datetime import datetime, timezone
 from pathlib import Path
 
 import rich_click as click
 
 from ..cli.utils import sql_dir_option
+from ..sensitivity import CODEOWNERS_FILE, check_paths, format_findings
 from ..util.common import block_coding_agents
 from .classification import runner, upstream
 from .classification.config import (
@@ -18,6 +21,10 @@
 )
 from .classification.runner import TargetKey
 
+# exit code CI keys on to request review; distinct from a tool crash (exit 1) so
+# the job knows to request DPE review rather than fail.
+SENSITIVITY_UNGATED_EXIT_CODE = 2
+
 
 def _parse_target(value: str) -> TargetKey:
     """Turn `project.dataset[.table]` into the triple the library takes."""
@@ -299,3 +306,50 @@ def classify(
         targets,
         refresh=refresh,
     )
+
+
+@data_governance.command()
+@click.argument("paths", nargs=-1, required=True, type=click.Path())
+@sql_dir_option
+@click.option(
+    "--codeowners",
+    default=CODEOWNERS_FILE,
+    help="CODEOWNERS file used to decide whether a flagged flow is already "
+    "owned/reviewed.",
+)
+@click.option(
+    "--json",
+    "as_json",
+    is_flag=True,
+    default=False,
+    help="Emit findings as JSON instead of the human-readable advisory.",
+)
+def sensitivity(paths, sql_dir, codeowners, as_json):
+    """Flag queries that read sensitive data and write it somewhere broader.
+
+    Scans the given query.sql files (or directories of them) for flows where a
+    restricted / narrowly workgroup-gated source is written to a more broadly
+    readable destination that the source's readers don't already cover.
+
+    Advisory and read-only: exits 2 when there are ungated flows so CI can
+    request @mozilla/dataplatform-wg review; it never blocks a merge.
+    """
+    findings = check_paths(list(paths), sql_dir, codeowners_file=codeowners)
+    ungated = [f for f in findings if f.get("gated") is False]
+
+    if as_json:
+        click.echo(json.dumps(findings, indent=2))
+    elif findings:
+        click.echo(format_findings(findings))
+
+    if ungated:
+        n = len({(f["source"], f["query"]) for f in ungated})
+        click.echo(
+            f"::warning::{n} sensitive-data flow(s) widen read access beyond the "
+            "source's authorized readers and aren't owned by any team. Requesting "
+            "@mozilla/dataplatform-wg review (advisory — this check does not "
+            "block).",
+            err=True,
+        )
+        sys.exit(SENSITIVITY_UNGATED_EXIT_CODE)
+    click.echo("no ungated sensitive-data flows", err=True)