diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..e6fe70a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,35 @@ +name: Publish to PyPI + +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Poetry + run: pip install poetry + - name: Build package + run: poetry build + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index eb7c8b2..131c381 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # shiny **/rsconnect-* tfbpshiny_log/ +notebooks/ +tfbpshiny_hf_cache/ # www/ **/www/*.js diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6d4627c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,33 @@ +# Changelog + +All notable changes to this project will be documented here. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + +## [1.0.0] - 2026-06-12 + +### Added + +- Initial public release of TFBPShiny. +- Dashboard interface for exploring transcription factor binding and perturbation + data from the Brent Lab yeast collection. +- Dataset Selection module with filter controls for binding and perturbation datasets. +- Binding module with correlation and scatter visualizations. +- Perturbation module with correlation and scatter visualizations. +- Comparison module with three subtabs: Compare Datasets (binding vs. perturbation + matrix), Compare Promoter Definitions (enrichment scores across four promoter sets: + Kang, Mindel, 500bp, Intergenic), and Compare Analysis Methods (promoter enrichment + vs. original peaks for ChIP-exo and ChEC-seq datasets). +- `python -m tfbpshiny launch` CLI entry point: downloads the HuggingFace dataset + cache on first run and serves the app on subsequent runs from the same directory. + Supports `--cache-dir`, `--skip-initialize`, `--no-materialize`, `--port`, `--host`, + and `--debug` flags. +- Projected in-memory materialization of dataset views at startup for improved query + performance; disabled via `--no-materialize` or `TFBPSHINY_MATERIALIZE=0`. +- Docker Compose production stack with Traefik reverse proxy and AWS CloudWatch + logging. +- shinyapps.io deployment support via `shinyapps_entry.py`. +- Terraform configuration for EC2 provisioning. diff --git a/CLAUDE.md b/CLAUDE.md index 48b3045..6160a0b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -160,7 +160,7 @@ Use the `vdb` instance to access data sources. Refer to the labretriever docs or poetry install # Run the application (development) -poetry run python -m tfbpshiny --log-level DEBUG shiny \ +poetry run python -m tfbpshiny --log-level DEBUG launch \ --port 8010 --host 127.0.0.1 --debug # Code quality diff --git a/README.md b/README.md index 0a476d7..f817540 100644 --- a/README.md +++ b/README.md @@ -5,262 +5,50 @@ data from the [Brent Lab yeast collection](https://huggingface.co/collections/Br --- -## Quick start (pip install) +## Resource Requirements -Install from GitHub into a virtual environment: +This app requires the following minimum resources to run: -```bash -python -m venv tfbpshiny-env -source tfbpshiny-env/bin/activate # Windows: tfbpshiny-env\Scripts\activate -pip install git+https://github.com/BrentLab/tfbpshiny@dev -``` - -Run the app: +- 4GB storage on disk +- 8GB RAM (10GB or more is recommended for better performance) -```bash -python -m tfbpshiny shiny -``` +## Quick start -Options: +If you wish to keep the app separated from your local environment, you should first +create a virtual environment. You can do this with `venv`. `cd` to the directory +where you want the virtual environment to be created, and run: ```bash -python -m tfbpshiny --log-level INFO shiny --port 8010 --host 127.0.0.1 +python -m venv tfbpshiny_env +source tfbpshiny_env/bin/activate ``` ---- - -## Production deployment - -### Prerequisites - -- An AWS account with permissions to create EC2 instances, IAM roles, - and security groups -- [Terraform](https://developer.hashicorp.com/terraform/install) ≥ 1.0 -- An EC2 key pair already created in `us-east-2` (or your target region) -- DNS A records for `tfbindingandperturbation.com`, - `www.tfbindingandperturbation.com`, - and `shinytraefik.tfbindingandperturbation.com` pointed at the - instance's public IP - -### 1. Provision the EC2 instance +### Install ```bash -cd terraform -cp terraform.tfvars.example terraform.tfvars -# Edit terraform.tfvars — set key_name and adjust instance_type / root_volume_gb -# if needed -terraform init -terraform apply +python -m pip install tfbpshiny ``` -Note the `public_ip` output and update your DNS records to point at it. +### Run the app: -### 2. Prepare the environment file - -The app requires a single `.env` file that is **not** stored in the repository. -Create it locally and copy it to the instance: - -#### .env - -```bash -DOCKER_ENV=true -HF_TOKEN= # optional; only for private HF datasets -VIRTUALDB_CONFIG=/path/to/config.yaml # optional; defaults to bundled config -TRAEFIK_DASHBOARD_PASSWORD_HASH=myusername:$$2y$$05$$... # see below -``` - -To generate the bcrypt hash for the Traefik dashboard: +This will download the necessary datasets from huggingface into a cache directory +that is created in your current working directory. By default, it is called +`./tfbpshiny_hf_cache`. When you run the app again, if you launch it from the same +location it will verify that the cache is up to date, and use it, without +re-downloading. You can also specify a custom cache directory with `--cache-dir`. ```bash -docker run --rm httpd:alpine htpasswd -nbB myusername mypassword -``` - -This prints something like: - -``` -myusername:$2y$05$abcdefghijklmnopqrstuuABCDEFGHIJKLMNOPQRSTUVWXYZ123456 +python -m tfbpshiny launch ``` -Copy the full output into `.env`, but **escape every `$` as `$$`** so Docker -Compose does not interpret them as variable references: +To install the latest development version from GitHub, use: ```bash -TRAEFIK_DASHBOARD_PASSWORD_HASH=myusername:$$2y$$05$$abcdefghijklmnopqrstuuABCDEFGHIJKLMNOPQRSTUVWXYZ123456 +python -m pip install git+https://github.com/BrentLab/tfbpshiny@dev ``` -Copy the env file to the instance: - -```bash -scp .env ec2-user@:/opt/tfbpshiny/ -``` - -### 3. Build and start the stack - -```bash -ssh ec2-user@ -cd /opt/tfbpshiny -docker compose -f production.yml up -d --build -``` - -**First deploy only** — fix `/hf-cache` volume ownership so the non-root `appuser` -can write HuggingFace downloads to the named volume: - -```bash -docker compose -f production.yml run --rm --user root shinyapp chown appuser /hf-cache -docker compose -f production.yml up -d -``` - -Traefik will automatically obtain a Let's Encrypt TLS certificate on first start. - -### HuggingFace cache - -The shinyapp container sets `HF_HOME=/hf-cache` and mounts a named Docker volume -there. HuggingFace model data is downloaded once and persists across container -rebuilds — no re-download on `docker compose up --build`. The volume ownership fix -above is only needed once; the volume retains correct permissions across rebuilds. - -### Logs - -Application and Traefik logs are sent to AWS CloudWatch Logs under the log group -`/tfbpshiny/production` in `us-east-2`. - ---- - -## shinyapps.io deployment - -This section describes how to deploy the app to -[shinyapps.io](https://www.shinyapps.io) as an alternative to the EC2/Docker -stack above. The two deployments are independent and can run in parallel. - -### Prerequisites - -- `rsconnect-python` installed: `pip install rsconnect-python` -- A HuggingFace token if any datasets are private - -### 1. Download the HuggingFace data locally - -The parquet files must be bundled with the deployment so the app never hits the -network on startup. Run the `initialize` command once, pointing at a directory -inside the project: - -```bash -HF_TOKEN= python -m tfbpshiny --cache-dir ./hf_cache initialize -``` - -**NOTE**: do call this hf_cache as it is already in the `.gitignore` - -This downloads all dataset parquet files into `hf_cache/` (~1.2 GB) and -verifies every view is readable. The directory is created relative to the -project root and will be included in the rsconnect upload bundle automatically. - -Re-run this command any time the upstream datasets are updated. - -### 2. Entry point - -`shinyapps_entry.py` in the project root is the shinyapps.io entry point. It -sets `HF_CACHE_DIR` to the bundled `hf_cache/` directory before importing the -Shiny app object, so no CLI flag is needed at runtime. No changes are required -— the file is already in the repository. - -### 3. Set environment variables in the dashboard - -In the shinyapps.io application dashboard under **Settings > Environment**, -add: - -| Variable | Value | -|---|---| -| `HF_TOKEN` | your HuggingFace token (if datasets are private) | - -Do not add `HF_CACHE_DIR` here — `shinyapps_entry.py` sets it from a path -relative to the bundle, which is more reliable than a hardcoded absolute path. - -### 4. Deploy - -**note**: Go to your shinyapps.io account, drop down the user menu, and go -to `Tokens`. If you click "show", and the python tab, it gives you this cmd with -the `name`, `account`, `token` and `secret` filled in. Run `rsconnect add` -once to store credentials under a nickname; subsequent deploys use `--name`. - -Generate a `requirements.txt` from the Poetry lockfile before deploying (rsconnect -requires it; it is gitignored because it is a generated artifact): - -```bash -poetry export --without-hashes --without dev -f requirements.txt -o requirements.txt -``` - -**NOTE**: the above must be done before uploading after any change to dependencies in -`pyproject.toml`. - -Make sure the local cache is up to date with the HuggingFace datasets, then deploy with rsconnect: - -```bash -poetry run python -m tfbpshiny --cache-dir ./hf_cache initialize -``` - -Then deploy with rsconnect: - -```bash -rsconnect add \ - --account \ - --name \ - --token \ - --secret - -CONNECT_REQUEST_TIMEOUT=3600 rsconnect deploy shiny . \ - --name \ - --entrypoint shinyapps_entry:app \ - --title "TF Binding and Perturbation" \ - --exclude "terraform" \ - --exclude "compose" \ - --exclude "tests" \ - --exclude "docs" \ - --exclude "tmp" \ - --exclude "data" \ - --exclude ".github" \ - --exclude ".vscode" \ - --exclude ".mypy_cache" \ - --exclude ".pytest_cache" \ - --exclude ".claude" \ - --exclude ".venv" \ - --exclude "mkdocs.yml" \ - --exclude "mkdocs_requirements.txt" \ - --exclude "production.yml" \ - --exclude "*.log" -``` - -rsconnect does not read `.gitignore`; it bundles everything it finds unless told -otherwise. The `--exclude` flags above strip deployment-irrelevant directories. -`hf_cache/` is intentionally not excluded — it is the bundled dataset cache and -must travel with the app. The first deploy uploads ~1.2 GB; set -`CONNECT_REQUEST_TIMEOUT` (seconds) high enough to cover the upload — 3600 (one -hour) is safe. - -### Updating the data - -When upstream datasets change, re-run step 1 and redeploy: - -```bash -HF_TOKEN= python -m tfbpshiny --cache-dir ./hf_cache initialize -CONNECT_REQUEST_TIMEOUT=3600 rsconnect deploy shiny . \ - --name \ - --entrypoint shinyapps_entry:app \ - --exclude "terraform" \ - --exclude "compose" \ - --exclude "tests" \ - --exclude "docs" \ - --exclude "tmp" \ - --exclude "data" \ - --exclude ".github" \ - --exclude ".vscode" \ - --exclude ".mypy_cache" \ - --exclude ".pytest_cache" \ - --exclude ".claude" \ - --exclude ".venv" \ - --exclude "mkdocs.yml" \ - --exclude "mkdocs_requirements.txt" \ - --exclude "production.yml" \ - --exclude "*.log" -``` +For production deployment (EC2/Docker) and shinyapps.io deployment instructions, +see [docs/development.md](docs/development.md). --- @@ -317,7 +105,7 @@ VIRTUALDB_CONFIG=/path/to/custom_config.yaml ### Running the app ```bash -poetry run python -m tfbpshiny --log-level DEBUG shiny \ +poetry run python -m tfbpshiny --log-level DEBUG launch \ --port 8010 --host 127.0.0.1 --debug ``` diff --git a/docs/development.md b/docs/development.md index 5f0c146..7697093 100644 --- a/docs/development.md +++ b/docs/development.md @@ -292,7 +292,7 @@ Plotly JS, or D3-based custom components. --- -## Production Deployment +## Production Deployment (EC2 / Docker) The production stack is defined in `production.yml` (Docker Compose). Two services: @@ -307,3 +307,223 @@ The production stack is defined in `production.yml` (Docker Compose). Two servic Both services use the AWS CloudWatch log driver. The EC2 instance and related infrastructure (security group, IAM role, user_data.sh bootstrap) are managed by Terraform in `terraform/`. + +### Prerequisites + +- An AWS account with permissions to create EC2 instances, IAM roles, + and security groups +- [Terraform](https://developer.hashicorp.com/terraform/install) >= 1.0 +- An EC2 key pair already created in `us-east-2` (or your target region) +- DNS A records for `tfbindingandperturbation.com`, + `www.tfbindingandperturbation.com`, + and `shinytraefik.tfbindingandperturbation.com` pointed at the + instance's public IP + +### 1. Provision the EC2 instance + +```bash +cd terraform +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars — set key_name and adjust instance_type / root_volume_gb +# if needed +terraform init +terraform apply +``` + +Note the `public_ip` output and update your DNS records to point at it. + +### 2. Prepare the environment file + +The app requires a single `.env` file that is **not** stored in the repository. +Create it locally and copy it to the instance: + +```bash +DOCKER_ENV=true +HF_TOKEN= # optional; only for private HF datasets +VIRTUALDB_CONFIG=/path/to/config.yaml # optional; defaults to bundled config +TRAEFIK_DASHBOARD_PASSWORD_HASH=myusername:$$2y$$05$$... # see below +``` + +To generate the bcrypt hash for the Traefik dashboard: + +```bash +docker run --rm httpd:alpine htpasswd -nbB myusername mypassword +``` + +This prints something like: + +``` +myusername:$2y$05$abcdefghijklmnopqrstuuABCDEFGHIJKLMNOPQRSTUVWXYZ123456 +``` + +Copy the full output into `.env`, but **escape every `$` as `$$`** so Docker +Compose does not interpret them as variable references: + +```bash +TRAEFIK_DASHBOARD_PASSWORD_HASH=myusername:$$2y$$05$$abcdefghijklmnopqrstuuABCDEFGHIJKLMNOPQRSTUVWXYZ123456 +``` + +Copy the env file to the instance: + +```bash +scp .env ec2-user@:/opt/tfbpshiny/ +``` + +### 3. Build and start the stack + +```bash +ssh ec2-user@ +cd /opt/tfbpshiny +docker compose -f production.yml up -d --build +``` + +**First deploy only** — fix `/hf-cache` volume ownership so the non-root `appuser` +can write HuggingFace downloads to the named volume: + +```bash +docker compose -f production.yml run --rm --user root shinyapp chown appuser /hf-cache +docker compose -f production.yml up -d +``` + +Traefik will automatically obtain a Let's Encrypt TLS certificate on first start. + +### HuggingFace cache + +The shinyapp container sets `HF_HOME=/hf-cache` and mounts a named Docker volume +there. HuggingFace data is downloaded once and persists across container rebuilds — +no re-download on `docker compose up --build`. The volume ownership fix above is +only needed once; the volume retains correct permissions across rebuilds. + +### Logs + +Application and Traefik logs are sent to AWS CloudWatch Logs under the log group +`/tfbpshiny/production` in `us-east-2`. + +--- + +## shinyapps.io Deployment + +This section describes how to deploy the app to +[shinyapps.io](https://www.shinyapps.io) as an alternative to the EC2/Docker +stack above. The two deployments are independent and can run in parallel. + +### Prerequisites + +- `rsconnect-python` installed: `pip install rsconnect-python` +- A HuggingFace token if any datasets are private + +### 1. Download the HuggingFace data locally + +The parquet files must be bundled with the deployment so the app never hits the +network on startup. Run `launch` once with a cache directory inside the project: + +```bash +HF_TOKEN= poetry run python -m tfbpshiny launch --cache-dir ./hf_cache --skip-initialize=false +``` + +**Note**: name the directory `hf_cache` — it is already in `.gitignore`. + +This downloads all dataset parquet files into `hf_cache/` (~1.2 GB) and verifies +every view is readable. The directory will be included in the rsconnect upload bundle +automatically. + +Re-run this command any time the upstream datasets are updated. + +### 2. Entry point + +`shinyapps_entry.py` in the project root is the shinyapps.io entry point. It sets +`HF_CACHE_DIR` to the bundled `hf_cache/` directory before importing the Shiny app +object, so no CLI flag is needed at runtime. No changes are required — the file is +already in the repository. + +### 3. Set environment variables in the dashboard + +In the shinyapps.io application dashboard under **Settings > Environment**, add: + +| Variable | Value | +|---|---| +| `HF_TOKEN` | your HuggingFace token (if datasets are private) | + +Do not add `HF_CACHE_DIR` here — `shinyapps_entry.py` sets it from a path relative +to the bundle, which is more reliable than a hardcoded absolute path. + +### 4. Deploy + +Go to your shinyapps.io account, drop down the user menu, and go to **Tokens**. Click +"show" on the Python tab — it gives the `rsconnect add` command with `name`, `account`, +`token`, and `secret` pre-filled. Run it once to store credentials under a nickname; +subsequent deploys use `--name`. + +Generate a `requirements.txt` from the Poetry lockfile before deploying (rsconnect +requires it; it is gitignored because it is a generated artifact): + +```bash +poetry export --without-hashes --without dev -f requirements.txt -o requirements.txt +``` + +Regenerate this file after any change to dependencies in `pyproject.toml`. + +Make sure the local cache is up to date, then deploy: + +```bash +rsconnect add \ + --account \ + --name \ + --token \ + --secret + +CONNECT_REQUEST_TIMEOUT=3600 rsconnect deploy shiny . \ + --name \ + --entrypoint shinyapps_entry:app \ + --title "TF Binding and Perturbation" \ + --exclude "terraform" \ + --exclude "compose" \ + --exclude "tests" \ + --exclude "docs" \ + --exclude "tmp" \ + --exclude "data" \ + --exclude ".github" \ + --exclude ".vscode" \ + --exclude ".mypy_cache" \ + --exclude ".pytest_cache" \ + --exclude ".claude" \ + --exclude ".venv" \ + --exclude "mkdocs.yml" \ + --exclude "mkdocs_requirements.txt" \ + --exclude "production.yml" \ + --exclude "*.log" +``` + +rsconnect does not read `.gitignore`; it bundles everything it finds unless told +otherwise. The `--exclude` flags above strip deployment-irrelevant directories. +`hf_cache/` is intentionally not excluded — it is the bundled dataset cache and must +travel with the app. The first deploy uploads ~1.2 GB; set `CONNECT_REQUEST_TIMEOUT` +(seconds) high enough to cover the upload — 3600 (one hour) is safe. + +### Updating the data + +When upstream datasets change, re-run step 1 and redeploy: + +```bash +HF_TOKEN= poetry run python -m tfbpshiny launch --cache-dir ./hf_cache --skip-initialize=false + +CONNECT_REQUEST_TIMEOUT=3600 rsconnect deploy shiny . \ + --name \ + --entrypoint shinyapps_entry:app \ + --exclude "terraform" \ + --exclude "compose" \ + --exclude "tests" \ + --exclude "docs" \ + --exclude "tmp" \ + --exclude "data" \ + --exclude ".github" \ + --exclude ".vscode" \ + --exclude ".mypy_cache" \ + --exclude ".pytest_cache" \ + --exclude ".claude" \ + --exclude ".venv" \ + --exclude "mkdocs.yml" \ + --exclude "mkdocs_requirements.txt" \ + --exclude "production.yml" \ + --exclude "*.log" +``` diff --git a/docs/sql_operations.md b/docs/sql_operations.md new file mode 100644 index 0000000..833a865 --- /dev/null +++ b/docs/sql_operations.md @@ -0,0 +1,718 @@ +# SQL Operations Reference + +Complete catalogue of every SQL operation executed by TFBPShiny. Each entry documents the +query's purpose, template, parameters, and result shape. The "Approximate result size" rows +are populated after running `notebooks/sql_audit.ipynb`. + +--- + +## Conventions + +- All table/view names are f-string interpolated — DuckDB cannot parameterize identifiers. +- Filter values use DuckDB `$name` syntax (named parameters). +- Functions that return `(sql, params)` without executing are marked **Builder** in the + Invocation column; the caller passes the result to `vdb.query(sql, **params)`. +- Functions that call `vdb.query()` internally are marked **Executes**. +- `sql_only=True` flag: where supported, returns `(sql, params)` instead of executing. + +--- + +## `select_datasets` module + +**File:** `tfbpshiny/modules/select_datasets/queries.py` +**Called from:** `server/workspace.py`, `server/sidebar.py`, `server/dataset_row.py` + +All nine functions in this file are **Builders** — they return `(sql, params)` and never +call `vdb.query()` themselves. + +--- + +### `metadata_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Fetch all sample-level metadata for one dataset, optionally filtered. + +```sql +SELECT * FROM {db_name}_meta +[WHERE {filter_clauses}] +``` + +**Parameters:** + +| Name | Type | Description | +|------|------|-------------| +| `cat_{field}` | list | Categorical filter: field must equal any value in list | +| `num_{field}_lo` / `_hi` | float | Numeric filter: BETWEEN lo AND hi | +| `bool_{field}` | bool | Boolean filter: field = value | + +**Result columns:** All columns in `{db_name}_meta` (schema varies by dataset). +**Approximate result size:** TBD + +--- + +### `sample_count_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Count how many samples (rows) match the current filters, optionally restricted +to a specific set of regulators. + +```sql +SELECT COUNT(sample_id) AS n FROM {db_name}_meta +[WHERE {filter_clauses}] +[AND regulator_locus_tag IN ($reg_{db_name}_0, $reg_{db_name}_1, ...)] +``` + +**Parameters:** Same filter params as `metadata_query`, plus optional `reg_{db_name}_{i}` +per regulator in `restrict_to_regulators`. + +**Result columns:** `n` (int — single row). +**Approximate result size:** Trivial (1 row × 1 col). + +--- + +### `regulator_locus_tags_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Return the distinct set of regulator locus tags present in a dataset after +applying filters. + +```sql +SELECT DISTINCT regulator_locus_tag FROM {db_name}_meta +[WHERE {filter_clauses}] +``` + +**Parameters:** Filter params as above. +**Result columns:** `regulator_locus_tag`. +**Approximate result size:** TBD + +--- + +### `regulator_display_labels_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Fetch the (locus_tag, symbol) pairs for a dataset, used to populate the +Regulator selectize in the filter modal. No parameters — returns all regulators. + +```sql +SELECT DISTINCT regulator_locus_tag, regulator_symbol +FROM {db_name}_meta +ORDER BY regulator_locus_tag +``` + +**Parameters:** None. +**Result columns:** `regulator_locus_tag`, `regulator_symbol`. +**Approximate result size:** TBD + +--- + +### `regulator_breakdown_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Determine whether a dataset has multiple samples per regulator (i.e., whether +an experimental-condition column is needed to disambiguate samples). Returns a single +aggregated row. + +```sql +WITH per_reg AS ( + SELECT regulator_locus_tag, + COUNT(DISTINCT "{col_1}") AS "{col_1}", + COUNT(DISTINCT "{col_2}") AS "{col_2}", + ... + FROM {db_name}_meta + [WHERE {filter_clauses}] + GROUP BY regulator_locus_tag + HAVING COUNT(*) > 1 +) +SELECT COUNT(*) AS n_multi, + COUNT(*) FILTER (WHERE "{col_1}" > 1) AS "{col_1}", + COUNT(*) FILTER (WHERE "{col_2}" > 1) AS "{col_2}", + ... +FROM per_reg +``` + +**Parameters:** Filter params as above. +**Result columns:** `n_multi` + one column per entry in `candidate_cols` (int counts). +**Approximate result size:** Trivial (1 row). + +--- + +### `matrix_diagonal_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Count distinct regulators and samples for every active dataset in one query. +Used to populate the diagonal cells of the dataset-selection matrix. + +```sql +SELECT '{db_name}' AS db_name, + COUNT(DISTINCT regulator_locus_tag) AS n_regulators, + COUNT(DISTINCT sample_id) AS n_samples +FROM {db_name}_meta +[WHERE {filter_clauses}] + +UNION ALL + +SELECT '{db_name_2}' AS db_name, ... +... +``` + +**Parameters:** One set of filter params per dataset, namespaced with `diag_{db_name}_`. +**Result columns:** `db_name`, `n_regulators`, `n_samples`. +**Approximate result size:** One row per active dataset (typically 6–10 rows). + +--- + +### `matrix_cross_dataset_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** For every (db_a, db_b) pair: count regulators in common and the restricted +sample counts for each side. Used to populate the off-diagonal cells of the dataset matrix. + +```sql +-- One subquery per pair, unioned: +SELECT * FROM ( + WITH common AS ( + SELECT regulator_locus_tag FROM {db_a}_meta [WHERE ...] + INTERSECT + SELECT regulator_locus_tag FROM {db_b}_meta [WHERE ...] + ) + SELECT '{db_a}__{db_b}' AS pair_id, + (SELECT COUNT(*) FROM common) AS n_common, + (SELECT COUNT(DISTINCT sample_id) + FROM {db_a}_meta [WHERE ...] AND regulator_locus_tag IN (SELECT ... FROM common)) AS samples_a, + (SELECT COUNT(DISTINCT sample_id) + FROM {db_b}_meta [WHERE ...] AND regulator_locus_tag IN (SELECT ... FROM common)) AS samples_b +) +UNION ALL +... +``` + +**Parameters:** Two sets of filter params per pair, with distinct prefixes (`cross_`, `cs_`). +**Result columns:** `pair_id`, `n_common`, `samples_a`, `samples_b`. +**Approximate result size:** One row per pair (N choose 2; typically 15 rows for 6 datasets). + +--- + +### `regulator_intersection_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Return the sorted list of regulator locus tags shared by two specific datasets +after applying their respective filters (excluding any regulator_locus_tag filter). Used to +populate the Regulator selector in a cross-dataset filter modal. + +```sql +SELECT regulator_locus_tag FROM {db_a}_meta [WHERE ...] +INTERSECT +SELECT regulator_locus_tag FROM {db_b}_meta [WHERE ...] +ORDER BY regulator_locus_tag +``` + +**Parameters:** Filter params for db_a (prefix `ri_{db_a}_`) and db_b (prefix `ri_{db_b}_`). +**Result columns:** `regulator_locus_tag`. +**Approximate result size:** TBD + +--- + +### `full_data_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Fetch all columns from the full data view (measurement + metadata joined) for +one dataset, optionally filtered. + +```sql +SELECT * FROM {db_name} +[WHERE {filter_clauses}] +``` + +**Parameters:** Filter params as for `metadata_query`. +**Result columns:** All columns in `{db_name}` (schema varies by dataset). +**Approximate result size:** TBD — potentially large (millions of rows for full datasets). + +--- + +## `binding` module + +**File:** `tfbpshiny/modules/binding/queries.py` +**Called from:** `server/workspace.py` + +--- + +### `binding_data_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Fetch per-target binding measurements for one dataset and measurement column. +Used as a subquery source by `corr_pair_sql` and `regulator_scatter_sql`. + +```sql +SELECT regulator_locus_tag, target_locus_tag, target_symbol, sample_id, {col} +FROM {db_name} +[WHERE {filter_clauses}] +``` + +**Parameters:** Filter params as above. +**Result columns:** `regulator_locus_tag`, `target_locus_tag`, `target_symbol`, `sample_id`, +`{col}` (measurement column, e.g. `enrichment`, `poisson_pval`). +**Approximate result size:** TBD + +--- + +### `corr_pair_sql` (binding) + +**Invocation:** Executes (or Builder when `sql_only=True`) +**sql_only path:** Yes +**Purpose:** Compute per-regulator Pearson or Spearman correlation between two binding +datasets. Returns one row per (regulator, sample_a, sample_b) combination. + +**Pearson template:** + +```sql +WITH + a_raw AS ( + SELECT regulator_locus_tag, target_locus_tag, target_symbol, sample_id, {col_a} + FROM {db_a} [WHERE ...] + ), + b_raw AS ( + SELECT regulator_locus_tag, target_locus_tag, target_symbol, sample_id, {col_b} + FROM {db_b} [WHERE ...] + ) +SELECT + '{db_a}' AS db_a, + a_raw.sample_id AS db_a_id, + '{db_b}' AS db_b, + b_raw.sample_id AS db_b_id, + a_raw.regulator_locus_tag, + corr(a_raw.{col_a}, b_raw.{col_b}) AS correlation +FROM a_raw +INNER JOIN b_raw + ON a_raw.regulator_locus_tag = b_raw.regulator_locus_tag + AND a_raw.target_locus_tag = b_raw.target_locus_tag +WHERE a_raw.{col_a} IS NOT NULL AND b_raw.{col_b} IS NOT NULL + AND NOT isinf(a_raw.{col_a}) AND NOT isinf(b_raw.{col_b}) + AND NOT isnan(a_raw.{col_a}) AND NOT isnan(b_raw.{col_b}) +GROUP BY a_raw.regulator_locus_tag, a_raw.sample_id, b_raw.sample_id +HAVING COUNT(*) >= 3 +``` + +**Spearman template:** Same structure, but with additional `ranked` CTE that applies +`RANK() OVER (PARTITION BY regulator, sample_a, sample_b ORDER BY ...)` before `corr()`. + +**Parameters:** Filter params for both datasets (namespaced with `{prefix}a_` / `{prefix}b_`). +**Result columns:** `db_a`, `db_a_id`, `db_b`, `db_b_id`, `regulator_locus_tag`, `correlation`. +**Approximate result size:** TBD + +--- + +### `corr_all_pairs_sql` (binding) + +**Invocation:** Executes +**sql_only path:** No +**Purpose:** Compute correlations for all active binding dataset pairs in a single query. +Wraps one `corr_pair_sql` subquery per pair in a `UNION ALL`. + +```sql +SELECT *, '{db_a}__{db_b}' AS pair_key FROM ( {corr_pair_sql for pair 0} ) +UNION ALL +SELECT *, '{db_a}__{db_b}' AS pair_key FROM ( {corr_pair_sql for pair 1} ) +... +``` + +**Parameters:** All per-pair parameters merged with pair-index prefix `p{i}_`. +**Result columns:** `db_a`, `db_a_id`, `db_b`, `db_b_id`, `regulator_locus_tag`, +`correlation`, `pair_key`. +**Approximate result size:** TBD + +--- + +### `regulator_scatter_sql` (binding) + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Fetch per-target values for a single regulator across two binding datasets, +suitable for rendering a scatter plot. Returns ranks for Spearman, raw values for Pearson. + +**Pearson template:** + +```sql +WITH a AS (SELECT ... FROM {db_a} WHERE regulator_locus_tag = $reg_a [AND ...]), + b AS (SELECT ... FROM {db_b} WHERE regulator_locus_tag = $reg_b [AND ...]) +SELECT a.target_locus_tag, + COALESCE(a.target_symbol, a.target_locus_tag) AS target_symbol, + a.{col_a} AS _val_a, + b.{col_b} AS _val_b +FROM a JOIN b ON a.target_locus_tag = b.target_locus_tag +``` + +**Spearman template:** Same, but wraps in an additional CTE with `RANK() OVER (...)` for +both columns. + +**Parameters:** Filter params (namespaced `rp{idx}a_` / `rp{idx}b_`) plus `rp{idx}reg_a` / +`rp{idx}reg_b` for the regulator locus tag. +**Result columns:** `target_locus_tag`, `target_symbol`, `_val_a`, `_val_b`. +**Approximate result size:** TBD (one row per shared target for one regulator, typically +hundreds of rows). + +--- + +## `perturbation` module + +**File:** `tfbpshiny/modules/perturbation/queries.py` +**Called from:** `server/workspace.py` + +The perturbation module uses the same `_corr_pair_sql_impl` as binding. The only difference +is `perturbation_data_query` replaces `binding_data_query` as the subquery builder. + +--- + +### `perturbation_data_query` + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Fetch per-target perturbation measurements for one dataset and measurement +column. Identical structure to `binding_data_query`. + +```sql +SELECT regulator_locus_tag, target_locus_tag, target_symbol, sample_id, {col} +FROM {db_name} +[WHERE {filter_clauses}] +``` + +**Parameters:** Filter params. +**Result columns:** `regulator_locus_tag`, `target_locus_tag`, `target_symbol`, `sample_id`, +`{col}`. +**Approximate result size:** TBD + +--- + +### `corr_pair_sql` (perturbation) + +**Invocation:** Executes (or Builder when `sql_only=True`) +**sql_only path:** Yes +**Purpose:** Per-regulator Pearson or Spearman correlation between two perturbation datasets. +Identical SQL structure to binding's `corr_pair_sql` — only the data source differs. + +See binding `corr_pair_sql` for full template. Replace `{db_a}` / `{db_b}` with perturbation +dataset names (e.g. `kemmeren`, `hackett`). + +**Result columns:** `db_a`, `db_a_id`, `db_b`, `db_b_id`, `regulator_locus_tag`, `correlation`. +**Approximate result size:** TBD + +--- + +### `corr_all_pairs_sql` (perturbation) + +**Invocation:** Executes +**sql_only path:** No +**Purpose:** Correlations for all active perturbation dataset pairs in one UNION ALL query. +Same structure as binding's `corr_all_pairs_sql`. + +**Result columns:** `db_a`, `db_a_id`, `db_b`, `db_b_id`, `regulator_locus_tag`, +`correlation`, `pair_key`. +**Approximate result size:** TBD + +--- + +### `regulator_scatter_sql` (perturbation) + +**Invocation:** Builder +**sql_only path:** N/A +**Purpose:** Per-target values for a single regulator across two perturbation datasets. +Identical SQL structure to binding's `regulator_scatter_sql`. + +**Result columns:** `target_locus_tag`, `target_symbol`, `_val_a`, `_val_b`. +**Approximate result size:** TBD (one row per shared target for one regulator). + +--- + +## `comparison` module + +**File:** `tfbpshiny/modules/comparison/queries.py` +**Called from:** `server/workspace.py` (inside `_run_analysis` extended task) + +--- + +### `fetch_dto_data` + +**Invocation:** Executes (or Builder when `sql_only=True`) +**sql_only path:** Yes +**Purpose:** Fetch the full DTO (Dual Transcription Overlap) empirical p-value table, +filtering to valid samples for each binding source. No user-controlled parameters. + +```sql +SELECT + d.binding_id_source, + d.perturbation_id_source, + d.dto_empirical_pvalue, + d.dto_fdr, + d.binding_set_size, + d.perturbation_set_size, + CAST(d.binding_id_id AS VARCHAR) AS binding_sample_id, + CAST(d.perturbation_id_id AS VARCHAR) AS pert_sample_id, + COALESCE(CAST(h.time AS VARCHAR), 'standard') AS time +FROM dto_expanded d +LEFT JOIN (SELECT DISTINCT sample_id, time FROM hackett_meta WHERE time = 45) h + ON d.perturbation_id_source = 'hackett' + AND CAST(d.perturbation_id_id AS VARCHAR) = CAST(h.sample_id AS VARCHAR) +LEFT JOIN (SELECT DISTINCT sample_id FROM callingcards) cc + ON d.binding_id_source = 'callingcards' + AND CAST(d.binding_id_id AS VARCHAR) = CAST(cc.sample_id AS VARCHAR) +LEFT JOIN (SELECT DISTINCT sample_id FROM harbison WHERE condition = 'YPD') harb + ON d.binding_id_source = 'harbison' + AND CAST(d.binding_id_id AS VARCHAR) = CAST(harb.sample_id AS VARCHAR) +WHERE + d.pr_ranking_column = 'log2fc' + AND (d.perturbation_id_source != 'hackett' OR h.sample_id IS NOT NULL) + AND (d.binding_id_source != 'callingcards' OR cc.sample_id IS NOT NULL) + AND (d.binding_id_source != 'harbison' OR harb.sample_id IS NOT NULL) +``` + +**Parameters:** None. +**Result columns:** `binding_id_source`, `perturbation_id_source`, `dto_empirical_pvalue`, +`dto_fdr`, `binding_set_size`, `perturbation_set_size`, `binding_sample_id`, `pert_sample_id`, +`time`. +**Approximate result size:** TBD + +--- + +### `topn_responsive_ratio` + +**Invocation:** Executes (or Builder when `sql_only=True`) +**sql_only path:** Yes +**Purpose:** For one (binding, perturbation) pair: compute the intersection of targets +present in both datasets, rank only the shared targets per binding sample, keep the top N, +then compute the fraction that are responsive (meet effect and p-value thresholds). +Intersecting before ranking ensures top-N slots are not consumed by binding targets absent +from the perturbation data. + +```sql +WITH binding AS ( + -- Harbison: MIN(pvalue) dedup across condition = 'YPD' + -- Others: direct SELECT with optional filters and target blacklist + SELECT + CAST({binding_sample_col} AS VARCHAR) AS binding_sample_id, + regulator_locus_tag, + target_locus_tag, + {rank_col} + FROM {binding_view} + [WHERE target_locus_tag NOT IN ($bl_bp{i}_0, ...) [AND {filter_clauses}]] +), +perturbation AS ( + SELECT + CAST(p.sample_id AS VARCHAR) AS perturbation_sample_id, + p.regulator_locus_tag, + p.target_locus_tag, + CASE WHEN ABS(p.{effect_col}) > $bp{i}_eff_thresh + AND p.{pvalue_col} < $bp{i}_pval_thresh + THEN 1 ELSE 0 END AS is_responsive + FROM {perturbation_view} p + [WHERE {filter_clauses}] +), +intersecting_targets AS ( + SELECT DISTINCT b.regulator_locus_tag, b.target_locus_tag + FROM binding b + INNER JOIN perturbation pert + ON b.regulator_locus_tag = pert.regulator_locus_tag + AND b.target_locus_tag = pert.target_locus_tag +), +binding_ranked AS ( + SELECT + b.binding_sample_id, + b.regulator_locus_tag, + b.target_locus_tag, + b.{rank_col}, + RANK() OVER ( + PARTITION BY b.binding_sample_id + ORDER BY b.{rank_col} {ASC|DESC} + ) AS rnk + FROM binding b + INNER JOIN intersecting_targets it + ON b.regulator_locus_tag = it.regulator_locus_tag + AND b.target_locus_tag = it.target_locus_tag + WHERE b.regulator_locus_tag != b.target_locus_tag +), +top_n_binding AS ( + SELECT binding_sample_id, regulator_locus_tag, target_locus_tag + FROM binding_ranked + WHERE rnk <= $bp{i}_top_n +) +SELECT + b.binding_sample_id, + b.regulator_locus_tag, + pert.perturbation_sample_id, + COUNT(*) AS n, + SUM(pert.is_responsive)::INTEGER AS n_responsive, + SUM(pert.is_responsive)::DOUBLE / COUNT(*) AS responsive_ratio +FROM top_n_binding b +JOIN perturbation pert + ON b.regulator_locus_tag = pert.regulator_locus_tag + AND b.target_locus_tag = pert.target_locus_tag +GROUP BY b.binding_sample_id, b.regulator_locus_tag, pert.perturbation_sample_id +``` + +**Per-dataset binding configuration:** + +| Dataset | `rank_col` | `rank_asc` | `binding_sample_col` | Dedup CTE | Blacklist | +|---------|-----------|-----------|----------------------|-----------|---------| +| `callingcards` | `poisson_pval` | True | `sample_id` | No | `CC_TARGET_BLACKLIST` | +| `callingcards_mindel` | `poisson_pval` | True | `sample_id` | No | `CC_TARGET_BLACKLIST` | +| `harbison` | `pvalue` | True | `sample_id` | Yes (MIN dedup, `condition='YPD'`) | No | +| `chec_m2025` | `enrichment` | False | `sample_id` | No | No | +| `chec_m2025_mindel` | `enrichment` | False | `sample_id` | No | No | +| `chec_m2025_peaks` | `peak_score` | False | `sample_id` | No | No | +| `rossi` | `enrichment` | False | `sample_id` | No | No | +| `rossi_mindel` | `enrichment` | False | `sample_id` | No | No | +| `rossi_peaks` | `peak_score` | False | `sample_id` | No | No | + +**Per-dataset responsiveness (default "Standard" preset):** + +| Dataset | `effect_col` | `pvalue_col` | Default thresholds | +|---------|-------------|-------------|-------------------| +| `degron` | `log2FoldChange` | `pvalue` | effect > 0, pval < 0.05 | +| `hughes_overexpression` | `mean_norm_log2fc` | (none) | effect > 0 | +| `hughes_knockout` | `mean_norm_log2fc` | (none) | effect > 0 | +| `kemmeren` | `Madj` | `pval` | effect > 0, pval < 0.05 | +| `hackett` | `log2_shrunken_timecourses` | (none) | effect > 0 | +| `hu_reimand` | `effect` | `pval` | effect > 0, pval < 0.05 | + +**Parameters:** See template above. `bp{i}_` prefix where `i` is the pair index in a batch. +**Result columns:** `binding_sample_id`, `regulator_locus_tag`, `perturbation_sample_id`, +`n`, `n_responsive`, `responsive_ratio`. +**Approximate result size:** TBD + +--- + +### `topn_all_pairs_sql` + +**Invocation:** Executes +**sql_only path:** No +**Purpose:** Compute `topn_responsive_ratio` for all (binding, perturbation) pairs in one +UNION ALL query. The main query run by the Comparison tab's Execute Analysis. + +```sql +SELECT *, '{b_db}__{p_db}' AS pair_key FROM ( {topn_responsive_ratio for pair 0} ) +UNION ALL +SELECT *, '{b_db}__{p_db}' AS pair_key FROM ( {topn_responsive_ratio for pair 1} ) +... +``` + +**Parameters:** All per-pair parameters merged with pair-index prefix `bp{i}_`. +**Result columns:** `binding_sample_id`, `regulator_locus_tag`, `perturbation_sample_id`, +`n`, `n_responsive`, `responsive_ratio`, `pair_key`. +**Approximate result size:** TBD + +--- + +## `utils` module + +--- + +### `_build_regulator_display_names` (startup DDL) + +**File:** `tfbpshiny/utils/vdb_init.py` +**Invocation:** Executes via `vdb._conn.execute()` (DDL, not `vdb.query()`) +**sql_only path:** No +**Called from:** `initialize_data()` at app startup +**Purpose:** Materialize a `regulator_display_names` DuckDB table from all registered +dataset meta views. Run once at startup; result persists for the session. + +```sql +CREATE OR REPLACE TABLE regulator_display_names AS +SELECT + regulator_locus_tag, + FIRST(regulator_symbol) AS regulator_symbol, + CASE + WHEN FIRST(regulator_symbol) IS NOT NULL + AND FIRST(regulator_symbol) != '' + AND FIRST(regulator_symbol) != FIRST(regulator_locus_tag) + THEN FIRST(regulator_symbol) || ' (' || regulator_locus_tag || ')' + ELSE regulator_locus_tag + END AS display_name +FROM ( + SELECT DISTINCT regulator_locus_tag, regulator_symbol FROM {db_1}_meta + UNION ALL + SELECT DISTINCT regulator_locus_tag, regulator_symbol FROM {db_2}_meta + ... +) __all +GROUP BY regulator_locus_tag +ORDER BY regulator_locus_tag +``` + +**Parameters:** None (db names are f-string interpolated). +**Result:** Creates `regulator_display_names` table with columns `regulator_locus_tag`, +`regulator_symbol`, `display_name`. +**Approximate result size:** TBD (one row per unique regulator across all datasets). + +--- + +### `get_regulator_display_name` + +**File:** `tfbpshiny/utils/vdb_init.py` +**Invocation:** Executes via `vdb._conn.execute()` directly +**sql_only path:** No +**Purpose:** Fetch display name entries from the pre-built lookup table. Used at module +startup to build the `{locus_tag: display_name}` dict for plot labels. + +```sql +-- All regulators: +SELECT * FROM regulator_display_names + +-- Filtered to specific tags: +SELECT * FROM regulator_display_names +WHERE regulator_locus_tag = ANY(?) +``` + +**Parameters:** Optional positional list `[locus_tags]` (DuckDB `?` syntax — not `$name`). +**Result columns:** `regulator_locus_tag`, `regulator_symbol`, `display_name`. +**Approximate result size:** TBD (same as `regulator_display_names` table). + +--- + +### `fetch_sample_condition_map` + +**File:** `tfbpshiny/utils/sample_conditions.py` +**Invocation:** Executes +**sql_only path:** No +**Purpose:** Build a `{sample_id: condition_label}` map for one dataset by querying its +`_meta` view for the condition columns that disambiguate multi-sample regulators. Column +names are double-quoted; identifier safety is checked before query construction. + +```sql +SELECT sample_id, "{cond_col_1}", "{cond_col_2}", ... +FROM {db_name}_meta +``` + +**Parameters:** None (all columns f-string interpolated). +**Result columns:** `sample_id` + one column per entry in `cols`. +**Post-processing:** Python iterates rows and calls `build_condition_label()` to produce +the string label. +**Approximate result size:** TBD (one row per sample in the dataset). + +--- + +## Parameter naming conventions + +| Pattern | Origin | Example | +|---------|--------|---------| +| `$cat_{field}` | `select_datasets._build_where` categorical | `$cat_Experimental_condition` | +| `$num_{field}_lo/hi` | `select_datasets._build_where` numeric | `$num_time_lo` | +| `$bool_{field}` | `select_datasets._build_where` boolean | `$bool_del_passed_qc` | +| `$cat_{prefix}{field}_{i}` | `binding/comparison._build_where` categorical (IN-list) | `$cat_bp0_b_poisson_pval_0` | +| `$num_{prefix}{field}_lo/hi` | `binding/comparison._build_where` numeric | `$num_bp0_b_time_lo` | +| `$bl_{prefix}_{i}` | `comparison.topn_responsive_ratio` target blacklist | `$bl_bp0_0` | +| `${prefix}_top_n` | `comparison.topn_responsive_ratio` top-N cutoff | `$bp0_top_n` | +| `${prefix}_eff_thresh` / `_pval_thresh` | `comparison.topn_responsive_ratio` responsiveness | `$bp0_eff_thresh` | +| `$diag_{db_name}_{field}` | `select_datasets.matrix_diagonal_query` | `$diag_rossi_time_lo` | +| `$cross_{pair}_{db}_{field}` | `select_datasets.matrix_cross_dataset_query` intersect arm | | +| `$cs_{pair}_{db}_{field}` | `select_datasets.matrix_cross_dataset_query` sample count arm | | +| `$ri_{db}_{field}` | `select_datasets.regulator_intersection_query` | | +| `$p{i}a_/b_` | `binding/perturbation._corr_pair_sql_impl` param namespace | `$p0a_cat_time_0` | +| `$rp{idx}a_/b_` | `binding/perturbation.regulator_scatter_sql` | | +| `$rp{idx}reg_a/b` | `binding/perturbation.regulator_scatter_sql` regulator | | diff --git a/pyproject.toml b/pyproject.toml index 37303cb..f9d9f17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tfbpshiny" -version = "0.0.0" +version = "1.0.0" description = "A shiny app which provides a dashboard to https://huggingface.co/collections/BrentLab/yeastresources" authors = ["chasem "] license = "GPL-3.0" diff --git a/shinyapps_entry.py b/shinyapps_entry.py index 4137ffc..bb6fb8a 100644 --- a/shinyapps_entry.py +++ b/shinyapps_entry.py @@ -15,5 +15,6 @@ from pathlib import Path os.environ["HF_CACHE_DIR"] = str(Path(__file__).parent / "hf_cache") +os.environ["TFBPSHINY_LOG_LEVEL"] = str(10) # logging.DEBUG from tfbpshiny.app import app # noqa: E402,F401 diff --git a/tests/unit/test_export.py b/tests/unit/test_export.py deleted file mode 100644 index 1a6c05b..0000000 --- a/tests/unit/test_export.py +++ /dev/null @@ -1,244 +0,0 @@ -"""Unit tests for the dataset export helpers.""" - -from __future__ import annotations - -import tarfile -from unittest.mock import MagicMock - -import duckdb -import pandas as pd -import pytest - -from tfbpshiny.modules.select_datasets.export import ( - ExportDataset, - _query_to_csv_bytes, - _safe_dir_name, - build_export_tarball, - build_readme, - get_dataset_description, -) - -# --- _safe_dir_name --- - - -@pytest.mark.parametrize( - "raw,expected", - [ - ("2026 Calling Cards", "2026_Calling_Cards"), - ("simple", "simple"), - ("has-hyphen", "has-hyphen"), - ("has.dot", "has.dot"), - ("../evil", "evil"), - ("../../etc/passwd", "etc_passwd"), - ("/absolute/path", "absolute_path"), - ("name\x00null", "name_null"), - ("a/b", "a_b"), - (" spaces ", "spaces"), - ("...", "dataset"), - ("___", "dataset"), - ("", "dataset"), - ], -) -def test_safe_dir_name(raw: str, expected: str) -> None: - assert _safe_dir_name(raw) == expected - - -# --- build_readme --- - - -def test_build_readme_includes_display_name(): - result = build_readme("2026 Calling Cards", "A binding dataset.") - assert "# 2026 Calling Cards" in result - - -def test_build_readme_includes_description(): - result = build_readme("Test", "Some description text.") - assert "Some description text." in result - - -def test_build_readme_includes_file_explanations(): - result = build_readme("Test", "desc") - assert "metadata.csv" in result - assert "annotated_features.csv" in result - assert "Sample-level metadata" in result - assert "Full genomic data" in result - - -# --- get_dataset_description --- - - -def test_get_dataset_description_delegates_to_vdb(): - """get_dataset_description passes through the return value of - vdb.get_dataset_description.""" - vdb = MagicMock() - vdb.get_dataset_description.return_value = "Harbison ChIP-chip data." - assert get_dataset_description(vdb, "harbison") == "Harbison ChIP-chip data." - vdb.get_dataset_description.assert_called_once_with("harbison") - - -def test_get_dataset_description_returns_none_when_vdb_returns_none(): - vdb = MagicMock() - vdb.get_dataset_description.return_value = None - assert get_dataset_description(vdb, "harbison") is None - - -# --- DuckDB-backed test fixtures --- - - -@pytest.fixture -def duckdb_vdb(): - """ - Minimal VirtualDB stub backed by a real in-memory DuckDB connection. - - Mirrors export.py's use of vdb._conn — update both if VirtualDB changes. - - """ - conn = duckdb.connect(":memory:") - conn.execute( - """ - CREATE TABLE test_meta AS - SELECT 's1' AS sample_id, 'BY4741' AS strain - UNION ALL - SELECT 's2', 'BY4741' - """ - ) - conn.execute( - """ - CREATE TABLE test_data AS - SELECT 's1' AS sample_id, 'YAL001C' AS gene, 1.0 AS value - UNION ALL - SELECT 's1', 'YAL002W', 2.0 - """ - ) - vdb = MagicMock() - vdb._conn = conn - yield vdb - conn.close() - - -def _open_tarball(buf): - """Open a BytesIO buffer as a tarball for reading.""" - buf.seek(0) - return tarfile.open(fileobj=buf, mode="r:gz") - - -# --- _query_to_csv_bytes --- - - -def test_query_to_csv_bytes(duckdb_vdb): - cursor = duckdb_vdb._conn.cursor() - csv_bytes = _query_to_csv_bytes(cursor, "SELECT * FROM test_meta", {}) - text = csv_bytes.decode("utf-8") - assert "sample_id" in text - assert "BY4741" in text - lines = text.strip().split("\n") - assert len(lines) == 3 # header + 2 rows - - -def test_query_to_csv_bytes_uses_lf_line_endings(duckdb_vdb): - """csv.writer must produce LF (not CRLF) for Unix tool compatibility.""" - cursor = duckdb_vdb._conn.cursor() - csv_bytes = _query_to_csv_bytes(cursor, "SELECT * FROM test_meta", {}) - assert b"\r\n" not in csv_bytes - assert b"\n" in csv_bytes - - -# --- build_export_tarball --- - - -def _make_export_dataset( - display_name: str = "Test Dataset", - description: str | None = None, -) -> ExportDataset: - return ExportDataset( - display_name=display_name, - metadata_sql="SELECT * FROM test_meta", - metadata_params={}, - data_sql="SELECT * FROM test_data", - data_params={}, - description=description, - ) - - -def test_single_dataset_with_description(duckdb_vdb): - ds = _make_export_dataset( - display_name="2026 Calling Cards", - description="A binding dataset.", - ) - buf = build_export_tarball([ds], duckdb_vdb) - - with _open_tarball(buf) as tar: - names = tar.getnames() - assert "2026_Calling_Cards/metadata.csv" in names - assert "2026_Calling_Cards/annotated_features.csv" in names - assert "2026_Calling_Cards/README.md" in names - - -def test_single_dataset_without_description(duckdb_vdb): - ds = _make_export_dataset(display_name="Test Dataset") - buf = build_export_tarball([ds], duckdb_vdb) - - with _open_tarball(buf) as tar: - names = tar.getnames() - assert "Test_Dataset/metadata.csv" in names - assert "Test_Dataset/annotated_features.csv" in names - assert "Test_Dataset/README.md" not in names - - -def test_multiple_datasets(duckdb_vdb): - ds1 = _make_export_dataset(display_name="Dataset A", description="First.") - ds2 = _make_export_dataset(display_name="Dataset B") - buf = build_export_tarball([ds1, ds2], duckdb_vdb) - - with _open_tarball(buf) as tar: - names = tar.getnames() - assert "Dataset_A/metadata.csv" in names - assert "Dataset_B/metadata.csv" in names - assert "Dataset_A/README.md" in names - assert "Dataset_B/README.md" not in names - - -def test_csv_content_matches_query(duckdb_vdb): - ds = _make_export_dataset(display_name="Check") - buf = build_export_tarball([ds], duckdb_vdb) - - with _open_tarball(buf) as tar: - meta_member = tar.extractfile("Check/metadata.csv") - assert meta_member is not None - recovered_meta = pd.read_csv(meta_member) - assert list(recovered_meta.columns) == ["sample_id", "strain"] - assert len(recovered_meta) == 2 - - data_member = tar.extractfile("Check/annotated_features.csv") - assert data_member is not None - recovered_data = pd.read_csv(data_member) - assert "gene" in recovered_data.columns - assert len(recovered_data) == 2 - - -def test_empty_dataset_list(duckdb_vdb): - buf = build_export_tarball([], duckdb_vdb) - with _open_tarball(buf) as tar: - assert tar.getnames() == [] - - -def test_tarball_no_path_traversal(duckdb_vdb): - """Verify that adversarial display_names never produce traversal paths.""" - ds = _make_export_dataset( - display_name="../../../etc/cron.d/backdoor", - ) - buf = build_export_tarball([ds], duckdb_vdb) - - with _open_tarball(buf) as tar: - for member in tar.getmembers(): - assert not member.name.startswith("/") - assert ".." not in member.name.split("/") - - -def test_progress_callback_called(duckdb_vdb): - """Verify the progress callback fires once per dataset.""" - ds1 = _make_export_dataset(display_name="A") - ds2 = _make_export_dataset(display_name="B") - names: list[str] = [] - build_export_tarball([ds1, ds2], duckdb_vdb, progress_callback=names.append) - assert names == ["A", "B"] diff --git a/tests/unit/test_matrix_namespacing.py b/tests/unit/test_matrix_namespacing.py new file mode 100644 index 0000000..e0cb93f --- /dev/null +++ b/tests/unit/test_matrix_namespacing.py @@ -0,0 +1,70 @@ +"""Unit tests: matrix cell buttons emit namespaced Shiny input ids. + +The matrix builders render plain ``