diff --git a/.devcontainer/codespaces-dev/devcontainer.json b/.devcontainer/codespaces-dev/devcontainer.json index 84812c4ccf..288df68f74 100644 --- a/.devcontainer/codespaces-dev/devcontainer.json +++ b/.devcontainer/codespaces-dev/devcontainer.json @@ -47,7 +47,8 @@ "vscode": { "extensions": [ "nf-core.nf-core-extensionpack", - "ms-vscode.live-server" + "ms-vscode.live-server", + "hashicorp.terraform" ], // Use Python from conda "settings": { diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index fa99bfe859..01ddd2ea77 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -27,7 +27,8 @@ "vscode": { "extensions": [ "nf-core.nf-core-extensionpack", - "ms-vscode.live-server" + "ms-vscode.live-server", + "hashicorp.terraform" ], // Use Python from conda "settings": { diff --git a/.devcontainer/install-platform-tools.sh b/.devcontainer/install-platform-tools.sh new file mode 100755 index 0000000000..646c164f0b --- /dev/null +++ b/.devcontainer/install-platform-tools.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +# Install the tooling used by the platform_automation side quest: +# - terraform : provision compute environments and pipelines declaratively +# - tw : the Seqera Platform CLI (tower-cli) +# - seqerakit : YAML-driven wrapper around tw for declarative Platform setup +# - az : the Azure CLI, used by Terraform's azurerm provider (az login) +# +# These run at container startup (via setup.sh -> onCreateCommand) rather than +# as devcontainer features. The production image (ghcr.io/nextflow-io/training) +# is only rebuilt on pushes to master, so feature additions don't reach an +# already-published image. Installing here makes the tools available in every +# container immediately. +# +# Idempotent (skips tools that are already present) and architecture-aware +# (amd64 + arm64) so re-runs and both CPU families are safe. +set -euo pipefail + +TERRAFORM_VERSION="1.9.8" +TW_VERSION="0.32.0" + +ARCH="$(uname -m)" + +# Make sure the download/extract tools the installs below rely on exist. +ensure_pkg() { + local cmd="$1" pkg="$2" + if ! command -v "${cmd}" >/dev/null 2>&1; then + echo "Installing ${pkg}..." + apt-get update -qq && apt-get install -y -qq "${pkg}" >/dev/null + fi +} + +ensure_pkg curl curl +ensure_pkg unzip unzip + +# --- Terraform ------------------------------------------------------------- +if command -v terraform >/dev/null 2>&1; then + echo "terraform already installed: $(terraform version | head -n1)" +else + case "${ARCH}" in + x86_64 | amd64) tf_arch="amd64" ;; + aarch64 | arm64) tf_arch="arm64" ;; + *) echo "Unsupported architecture for terraform: ${ARCH}" >&2; exit 1 ;; + esac + echo "Installing Terraform ${TERRAFORM_VERSION} (${tf_arch})..." + tmp="$(mktemp -d)" + curl -fSL "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_${tf_arch}.zip" -o "${tmp}/terraform.zip" + unzip -q "${tmp}/terraform.zip" -d "${tmp}" + install -m 0755 "${tmp}/terraform" /usr/local/bin/terraform + rm -rf "${tmp}" +fi + +# --- Seqera CLI (tw) ------------------------------------------------------- +# tower-cli only publishes a native Linux binary for x86_64; on arm64 install +# the architecture-independent JAR (Java is provided by the java feature). +if command -v tw >/dev/null 2>&1; then + echo "tw already installed: $(tw --version 2>/dev/null | head -n1)" +else + base="https://github.com/seqeralabs/tower-cli/releases/download/v${TW_VERSION}" + case "${ARCH}" in + x86_64 | amd64) + echo "Installing Seqera CLI (tw) ${TW_VERSION} native binary for x86_64..." + curl -fSL "${base}/tw-linux-x86_64" -o /usr/local/bin/tw + chmod +x /usr/local/bin/tw + ;; + aarch64 | arm64) + echo "Installing Seqera CLI (tw) ${TW_VERSION} JAR for arm64..." + mkdir -p /usr/local/lib/tw + curl -fSL "${base}/tw-jar.jar" -o /usr/local/lib/tw/tw.jar + cat > /usr/local/bin/tw <<'EOF' +#!/usr/bin/env bash +exec java -jar /usr/local/lib/tw/tw.jar "$@" +EOF + chmod +x /usr/local/bin/tw + ;; + *) echo "Unsupported architecture for tw: ${ARCH}" >&2; exit 1 ;; + esac +fi + +# --- seqerakit ------------------------------------------------------------- +# Pure-Python (arch-independent) wrapper that drives tw from YAML. Installed +# with the conda pip (python.defaultInterpreterPath = /opt/conda/bin/python), +# so no PEP 668 override is needed. Depends on tw being on PATH (installed above). +if command -v seqerakit >/dev/null 2>&1; then + echo "seqerakit already installed: $(seqerakit --version 2>/dev/null | head -n1)" +else + echo "Installing seqerakit..." + pip install --quiet seqerakit +fi + +# --- Azure CLI (az) -------------------------------------------------------- +# Microsoft's deb installer sets up the apt repo and pulls the right package +# for the host architecture (amd64 + arm64 are both published). +if command -v az >/dev/null 2>&1; then + echo "az already installed: $(az version --output tsv --query '"azure-cli"' 2>/dev/null | head -n1)" +else + echo "Installing Azure CLI (az)..." + curl -sL https://aka.ms/InstallAzureCLIDeb | bash +fi diff --git a/.devcontainer/local-dev/devcontainer.json b/.devcontainer/local-dev/devcontainer.json index 130b120d66..0d7a262372 100644 --- a/.devcontainer/local-dev/devcontainer.json +++ b/.devcontainer/local-dev/devcontainer.json @@ -49,7 +49,8 @@ "vscode": { "extensions": [ "nf-core.nf-core-extensionpack", - "ms-vscode.live-server" + "ms-vscode.live-server", + "hashicorp.terraform" ], // Use Python from conda "settings": { diff --git a/.devcontainer/local-features/uv-tools/install.sh b/.devcontainer/local-features/uv-tools/install.sh index 8537c04973..d6465a6be7 100644 --- a/.devcontainer/local-features/uv-tools/install.sh +++ b/.devcontainer/local-features/uv-tools/install.sh @@ -5,3 +5,5 @@ uv tool install pre-commit uv tool install nf-core==3.5.2 uv tool install "mkdocs-quiz>=1.5.2" +# seqerakit: used by the platform_automation side quest +uv tool install "seqerakit==0.5.7" diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 0811dfc1c1..5ebc9c136c 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -30,4 +30,7 @@ cd /workspaces/training rm -rf /tmp/nf-test ~/.m2/repository echo "nf-test updated with v2 parser support" +# Install the platform_automation side quest tools (terraform, tw, az) +bash "$(dirname "$0")/install-platform-tools.sh" + cat /usr/local/etc/vscode-dev-containers/first-run-notice.txt diff --git a/.editorconfig b/.editorconfig index 5e6d11b4b4..252370594c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,6 +11,10 @@ indent_style = space [*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 +# Terraform's canonical `terraform fmt` style uses 2-space indentation +[*.{tf,tfvars}] +indent_size = 2 + # ignore python and markdown [*.{py,md}] indent_style = unset diff --git a/.gitignore b/.gitignore index 68936ef169..9bf9616195 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,17 @@ _scripts/.venv/ # Tutorial working directories (created by learners during exercises) hello-nf-core/core-hello/ + +# Terraform (platform_automation side quest) +**/.terraform/* +.terraform.lock.hcl +*.tfstate +*.tfstate.* +*.tfvars +!*.tfvars.example +crash.log +crash.*.log + +# Agent/tooling local state +.omc/ +**/.claude/.local/ diff --git a/docs/en/docs/side_quests/co_scientist/01_meet_coscientist.md b/docs/en/docs/side_quests/co_scientist/01_meet_coscientist.md new file mode 100644 index 0000000000..d895d33bd9 --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/01_meet_coscientist.md @@ -0,0 +1,109 @@ +# Meet CoScientist + +Working on the Seqera Platform means moving between the Launchpad, compute environments, datasets, and runs. +CoScientist can drive all of that from a conversation, once it is connected to your workspace. +In this lesson you open the chat, connect it to your training workspace, and have it inspect and register real Platform assets, so the rest of the course has a working agent to build on. + +--- + +## 1. Open the chat and connect to your workspace. + +Navigate to [https://ai.seqera.io/chat](https://ai.seqera.io/chat) and sign in with your Seqera credentials. + +![The CoScientist chat after signing in](img/chat-landing.png) + +Use the **Workspace** selector in the top navigation to choose the provided training workspace. + +## 2. Confirm the connection and survey the workspace. + +!!! tip "Writing good prompts" + + Be specific and state the action or output you want. + [Working with the agent](02_working_with_the_agent.md) covers prompting for intent in depth. + +Send one prompt to confirm CoScientist can see your workspace and to get a snapshot of what is already in it: + +```text +Which Seqera workspace am I connected to? List the compute environments, the pipelines on the Launchpad, and the datasets you can see. +``` + +??? example "What CoScientist typically does" + + It names the connected workspace and lists the compute environment(s), Launchpad pipelines, and datasets it can see. + The exact wording and the number of items will differ depending on the workspace state. + +The same access also covers reference genomes and data links. + +!!! note "Checkpoint" + + CoScientist correctly names the **training workspace** and lists at least one compute environment, along with any pipelines and datasets (or reports that none exist yet). + If it cannot name the workspace, the connection is not set up; revisit step 1. + +## 3. Ask CoScientist what it can do. + +Get a baseline picture of the agent's capabilities in this workspace: + +```text +What can you help me with in this workspace? What can you see and what actions can you take? +``` + +??? example "What CoScientist typically does" + + It describes that it can help develop pipelines, launch and monitor runs, browse data, and act on GitHub. + The exact wording will differ from run to run. + +## 4. Add rnaseq-nf to the Launchpad. + +Ask CoScientist to register a pipeline on your behalf: + +```text +Add the pipeline https://github.com/nextflow-io/rnaseq-nf to the Launchpad in this workspace, using the available compute environment and a tag to identify it as created by me. +``` + +CoScientist will confirm the action or ask which compute environment to use if more than one is available. + +!!! note "Checkpoint" + + In the Seqera Platform web app, open **Launchpad**: an entry for `rnaseq-nf` is now present. + +![rnaseq-nf on the Launchpad](img/launchpad-rnaseq-nf.png) + +## 5. Understand what rnaseq-nf does. + +`rnaseq-nf` is a small RNA-seq quantification pipeline, a good size to develop and test against. +It builds a Salmon index from a transcriptome (`INDEX`), runs quality control on the reads (`FASTQC`), quantifies transcript expression with Salmon (`QUANT`), and aggregates the results into a single report (`MULTIQC`). +It ships a small chicken (`ggal`) test dataset, so a full run finishes in minutes. + +```mermaid +graph LR + T[Transcriptome] --> INDEX[INDEX
salmon index] + R[Reads] --> FASTQC[FASTQC
quality control] + R --> QUANT[QUANT
salmon quant] + INDEX --> QUANT + FASTQC --> MULTIQC[MULTIQC
report] + QUANT --> MULTIQC + MULTIQC --> OUT[multiqc_report.html] +``` + +## 6. Inspect the compute environment. + +Ask CoScientist to read the configuration for the compute environment attached to the pipeline: + +```text +Show me the details of the compute environment this pipeline will run on. +``` + +CoScientist reads the compute-environment configuration and returns the key fields. + +!!! note "Checkpoint" + + CoScientist reports the compute environment name and platform (for example AWS Batch) matching the training compute environment. + +### Takeaway + +You connected CoScientist to your training workspace, had a first conversation, and used it to register and inspect a pipeline. +It acts on real Platform assets on your behalf, with your own permissions. + +### What's next? + +In the next lesson, [learn how to work effectively with the agent](02_working_with_the_agent.md) before you start changing code. diff --git a/docs/en/docs/side_quests/co_scientist/02_working_with_the_agent.md b/docs/en/docs/side_quests/co_scientist/02_working_with_the_agent.md new file mode 100644 index 0000000000..f874d2a7cf --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/02_working_with_the_agent.md @@ -0,0 +1,103 @@ +# Working with the agent + +CoScientist takes real actions on your workspace, so how you prompt and check it matters. +In this lesson you launch a run that fails on a simple mistake, then drive the agent to fix it from the user side, without editing any pipeline code. +Along the way you practise the habits that keep you in control: prompt for intent, verify what the agent did, and redirect it when it goes wrong. + +--- + +## 1. Launch a run that will fail. + +`rnaseq-nf` ships its own test transcriptome and uses it by default. +To create a failure to work with, launch the pipeline but override that default with a path that does not exist: + +```text +Launch my rnaseq-nf on the Launchpad, but set the transcriptome parameter to /tmp/missing.fa so we have a failed run to work with. +``` + +The run fails almost immediately at the indexing step, because the file you pointed it at is not there. + +!!! note "Checkpoint" + + A run for rnaseq-nf appears in the Runs list and fails quickly with a file-not-found error. + +## 2. Open the failed run from the Platform. + +CoScientist works in both directions. +You launched this run from the chat; you can also reach the agent from a run in the Platform. +Open the **Runs** list, select the failed run, and choose **Explain with AI**. +This starts a CoScientist conversation loaded with that run's context, so you do not have to describe the failure yourself. + +![The Explain with AI button on a failed run](img/explain-with-ai.png) + +!!! note "Checkpoint" + + Selecting **Explain with AI** opens a CoScientist conversation about the failed run. + +## 3. Prompt for intent. + +Ask the agent to work the problem, giving it the symptom and a clear goal rather than a vague instruction. + +=== "Better" + + ```text + This rnaseq-nf run failed. Read the run log, tell me the cause, and propose how to fix the launch settings before changing anything. + ``` + +=== "Vague" + + ```text + Fix the pipeline. + ``` + +The better prompt points the agent at the run log, asks it to explain before acting, and limits it to the launch settings. +The vague one invites it to guess, and the guess is often wrong. + +??? example "What CoScientist typically does" + + It reads the run log, reports that the transcriptome file could not be found, and proposes correcting the parameter to the default path. + The exact wording will differ from run to run. + +## 4. Verify what the agent says. + +Do not take the agent's diagnosis, or its "Done", at face value. +Check its explanation against the actual run error, and confirm the cause is the parameter you set, not a problem in the pipeline. + +!!! warning + + Treat the agent's "Done" as a claim to check, not a fact. + The Checkpoints in this side quest exist for this reason: each one is a real Platform state you confirm yourself, whatever the agent says it did. + +!!! note "Checkpoint" + + From the run's own error, you have confirmed the failure is the missing transcriptome file you set at launch. + +## 5. Redirect the agent when it goes too far. + +The fix here is a launch parameter, not a code change. +The default transcriptome ships with the pipeline, so fixing the run means dropping your override, not finding new data. +If the agent offers to edit the pipeline, steer it back to the smallest fix: + +```text +Don't change the pipeline code. Just remove the transcriptome override so the pipeline uses its default, and relaunch. +``` + +!!! note "Checkpoint" + + The agent corrects only the launch parameter and leaves the pipeline code untouched. + +## 6. Re-launch and confirm the fix. + +With the parameter corrected, the run gets past the step that failed. + +!!! note "Checkpoint" + + The new run launches and proceeds past the indexing step instead of failing immediately. + +### Takeaway + +You fixed a failed run entirely from the user side, without touching pipeline code, by prompting for intent, verifying the agent's diagnosis against the real error, and redirecting it to the smallest fix. + +### What's next? + +In the next lesson, [start developing with CoScientist](03_develop_with_coscientist.md), where you change the pipeline itself. diff --git a/docs/en/docs/side_quests/co_scientist/03_develop_with_coscientist.md b/docs/en/docs/side_quests/co_scientist/03_develop_with_coscientist.md new file mode 100644 index 0000000000..2d52352560 --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/03_develop_with_coscientist.md @@ -0,0 +1,69 @@ +# Develop with CoScientist + +Adding a step to a pipeline normally means finding a tool, writing a process, wiring its inputs and outputs, and getting the change reviewed. +CoScientist can do that legwork for you, and the developer journey starts here, still in the web chat. +You fork `rnaseq-nf`, give CoScientist scoped access to your fork, and have it add a new process by reusing an nf-core module and open a pull request, letting it work until the change is done. + +--- + +## 1. Fork rnaseq-nf. + +Fork the pipeline into your own GitHub account so you have a copy to change, and so you can scope CoScientist's access to just this repository. +On GitHub, open [nextflow-io/rnaseq-nf](https://github.com/nextflow-io/rnaseq-nf) and select **Fork**. + +!!! note "Checkpoint" + + The repository `your-user/rnaseq-nf` exists on your GitHub account. + +## 2. Give CoScientist access to your fork. + +CoScientist commits changes and opens pull requests on your behalf using a GitHub token. +Create a **fine-grained personal access token** scoped to only your fork, rather than a broad token, so the agent can touch nothing else. + +On GitHub, go to **Settings → Developer settings → Personal access tokens → Fine-grained tokens** and generate a token with: + +- **Repository access**: only your `rnaseq-nf` fork +- **Permissions**: Contents (read and write) and Pull requests (read and write) + +Add the token to your CoScientist session by clicking your name at the bottom left and selecting **GitHub access token**. +After adding the token, start a new session so CoScientist picks it up; an existing session keeps using the old token. + +!!! note "Checkpoint" + + CoScientist reports that it can access your fork. + +## 3. Add a trimming step and open a pull request. + +Ask CoScientist to add a `fastp` read-trimming step before quantification, reusing the nf-core fastp module, and to open a pull request with the change: + +```text +In my fork of rnaseq-nf, add a fastp read-trimming step before quantification, reusing the nf-core fastp module, wire its trimmed reads into the QUANT process, and open a pull request. My GitHub user is ''. +``` + +Reusing the nf-core module keeps the work small: CoScientist pulls in a tested process and mainly has to get the input and output wiring right, rather than reimplementing fastp. +This exercises its nf-core module discovery. + +??? example "What CoScientist typically does" + + It finds the nf-core/fastp module, adds it under `modules/nf-core/fastp/`, wires the trimmed reads into the workflow, commits to a branch on your fork, and opens a pull request titled something like "add fastp trimming step before quantification". + The exact wording will differ from run to run. + +## 4. Let the agent work until it is done. + +Adding a process rarely lands in one attempt. +Let CoScientist iterate: when something is wrong it adjusts the wiring and tries again, until the change is complete and the pull request is open. +Keep it on track with the habits from the previous lesson: check what it changed, and redirect it if it goes too far. + + + +!!! note "Checkpoint" + + A pull request with the `fastp` step is open on your fork. + +### Takeaway + +Working entirely in the web chat, you connected GitHub, forked `rnaseq-nf`, and had CoScientist add a real processing step by reusing an nf-core module and open a pull request. + +### What's next? + +In the next lesson, [move to the CLI](04_move_to_the_cli.md) to test the change and see why the CLI suits development work. diff --git a/docs/en/docs/side_quests/co_scientist/04_move_to_the_cli.md b/docs/en/docs/side_quests/co_scientist/04_move_to_the_cli.md new file mode 100644 index 0000000000..72969c31fd --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/04_move_to_the_cli.md @@ -0,0 +1,145 @@ +# Move to the CLI + +You developed the change and opened the pull request in the web chat. +You can keep working there, but running and iterating on tests is faster in your own terminal, so now you move to the `seqera ai` command-line agent. + +--- + +## 1. Why move to the CLI. + +Developing and testing a pipeline is better suited to your own terminal than the browser: + +- Compute: the CLI uses your machine, which has more resources than the in-browser environment. +- Docker: it is available locally, so containerized processes and test runs work. +- Local access: the agent works against your own files, repos, editor, and personal credentials and config. +- Iteration: you edit, run, and see results in one terminal, without round-tripping through the web chat. + +## 2. Open your fork in the devcontainer. + +You forked `rnaseq-nf` in the previous lesson, and CoScientist opened a pull request with the `fastp` step. +Open your fork in the devcontainer, which comes with the tools you need already installed. +Then switch to the pull request's branch so you can run and test the change: + +```bash +git checkout # CoScientist names it like seqera-co-scientist/add-fastp-trimming-... +``` + + + +!!! note "Checkpoint" + + You are in the devcontainer for your fork, on the branch with the `fastp` change. + +## 3. Install and start the CLI. + +The devcontainer already includes the `seqera` CLI. +To install it yourself, or to work outside the devcontainer, run the install script: + +```bash +curl -fsSL https://ai.seqera.io/install | bash +``` + +Then authenticate with a Seqera Platform access token. +Create one at [cloud.seqera.io/tokens](https://cloud.seqera.io/tokens) and export it before starting the CLI: + +```bash +export TOWER_ACCESS_TOKEN= +``` + +The Seqera AI credits are charged on a **per-organization** basis, so you must select the right organization before using the + +```bash +seqera org switch +``` + +The interactive prompt will tell you which org to select + +```console +Available organizations: + + 1. community (pro) + 2. Your-Org + +Enter selection (1-2), or 'q' to cancel: +``` + +!!! note "Checkpoint" + + `seqera ai` is running in your local clone, configured to use your Seqera Platform workspace AI credits. + +## 4. Ask for an nf-test and run it to green. + +`rnaseq-nf` has no `nf-test` coverage. +If you are new to `nf-test`, see [Testing with nf-test](../nf_test/index.md) for background on snapshot assertions. + +The simplest place to start is a pipeline-level test that runs the whole pipeline on its test data and checks the outputs. +Before writing the prompt, decide what is safe to assert on: not all output is stable, and snapshotting unstable output causes tests to fail on every run for reasons unrelated to correctness. + +| Output | Snapshot? | Why | +| ----------------------------------------------------- | --------- | ------------------------------------------------------- | +| `quant_/quant.sf` (Salmon per-transcript counts) | Yes | Deterministic counts for fixed inputs and version | +| Output file and directory existence | Yes | Structural, stable | +| `quant_/cmd_info.json`, `aux_info/meta_info.json` | No | Embed the command line, Salmon version, and metadata | +| `quant_/logs/salmon_quant.log` | No | Contains timestamps and runtimes | +| `multiqc_report.html` | No | Embeds a timestamp, tool versions, and an absolute path | +| FastQC `*_fastqc.zip` / `*_fastqc.html` | No | Zips embed timestamps and the FastQC version | +| Any path containing the work-directory hash | No | Changes every run | + +!!! tip + + Salmon can introduce tiny nondeterminism across threads. + Run it single-threaded (`--threads 1`) when you want a byte-stable `quant.sf` to snapshot. + +With those rules in mind, ask CoScientist to create the test, telling it what to snapshot and what to leave out: + +```text +Add a pipeline-level nf-test that runs the whole pipeline on its test data. Assert on the quant.sf columns and that the expected output files exist; do not snapshot the MultiQC HTML, Salmon logs, cmd_info.json, or anything containing timestamps, versions, or work directory paths. +``` + +??? example "What CoScientist typically does" + + It scaffolds a `tests/` directory and a pipeline-level `.nf.test` that runs the pipeline end to end and asserts on the stable outputs. + The exact wording will differ from run to run. + +For a reference of what the generated test looks like, see [`solutions/pipeline.nf.test`](solutions/pipeline.nf.test). + +Then use **goal mode** to run the test repeatedly until it passes, fixing the test along the way: + +```text +/goal run the nf-tests until they pass and fix any issues in the tests +``` + +Goal mode keeps working toward an objective across several attempts and stops once the goal is met. +The agent runs `nf-test`, reads any failure, narrows the assertion or updates the snapshot, and re-runs, repeating until the test passes. +By default the CLI asks for your approval before it runs a command, so you see each `nf-test` invocation before it executes. + +!!! note "Checkpoint" + + `nf-test test` reports a passing test for the pipeline. + +## 5. Add the tests and CI to your pull request. + +A CI workflow runs the tests automatically on every change, so a regression is caught before it merges. +The pull request you opened earlier contains the `fastp` step. +Ask the agent to create the workflow and commit it and the tests to the same branch, so the pull request picks them up, in one prompt: + +```text +Add a GitHub Actions workflow that installs nf-test and runs the test suite on every push and pull request, then commit the workflow and the nf-test to the branch of my open pull request and push. +``` + + + +!!! note "Checkpoint" + + A workflow file exists under `.github/workflows/` that runs `nf-test`, and the open pull request now contains the `fastp` step, the tests, and the workflow. + +Open the pull request and confirm the diff contains the `fastp` step, the tests, and the workflow, and that it targets the branch you intend. + +### Takeaway + +You moved to the CLI, added test coverage the pipeline lacked, chose what was safe to snapshot, and wired the tests into CI. +All of it joined the pull request you opened from the web chat. + +### What's next? + +[Package this into a reusable skill](05_build_a_skill.md). diff --git a/docs/en/docs/side_quests/co_scientist/05_build_a_skill.md b/docs/en/docs/side_quests/co_scientist/05_build_a_skill.md new file mode 100644 index 0000000000..7f5d520362 --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/05_build_a_skill.md @@ -0,0 +1,88 @@ +# Build a reusable skill + +In the last lesson you hand-wrote a pipeline-level `nf-test` and learned which output is safe to snapshot. +Capture that discipline as a reusable skill, so CoScientist applies the same rules in every session without you re-explaining them. + +--- + +## 1. Enhance CoScientist with your own skills. + +CoScientist ships with built-in skills, and you can add your own. +A skill is a small instruction set that CoScientist loads as session context, so it follows your guidance automatically whenever you ask for related work. +Packaging the testing rules you just applied makes a good first skill. + +## 2. Author the skill. + +A skill lives in its own directory with a `SKILL.md` file: YAML frontmatter with a `name` and a `description` (both required, or the skill is skipped), followed by a markdown body of instructions. +CoScientist discovers skills from your project and user skill directories and sends them to the session as context. +A project directory like `.agents/skills/` takes priority, so a repository can override a global skill. + +Create the directory and the empty skill file: + +```bash +mkdir -p .agents/skills/write-nf-test +touch .agents/skills/write-nf-test/SKILL.md +``` + +Then open `.agents/skills/write-nf-test/SKILL.md` and add this content: + +```markdown +--- +name: write-nf-test +description: Generate an nf-test that asserts on stable output and excludes unstable content. +--- + +# Write nf-test + +When asked to write an nf-test for a process or a pipeline, follow the rules below. + +## Steps + +1. Scaffold the test under `tests/`. + For a process use a `nextflow_process` block; for a pipeline use a `nextflow_pipeline` block. + Provide representative inputs or test data so the test runs end to end. + +2. Assert on deterministic output only: + + - Numeric or tabular result files, such as a Salmon `quant.sf` table + - The existence of the expected output files and directories + +3. Do NOT snapshot unstable content: + + - Reports that embed timestamps or versions, such as MultiQC HTML + - Log files, and files that record the command line or tool version (for example Salmon `cmd_info.json` and `meta_info.json`) + - Any path or value containing the work-directory hash + - Version strings + +4. Run `nf-test test` on the generated file. + When it fails, narrow the assertion rather than snapshotting unstable content, and re-run until it passes. +``` + +Keep skills small: CoScientist caps each discovered skill's context at 5 KB. +The `.agents/skills/` location follows the cross-agent Agent Skills convention, so the same skill works in other compatible agents. + +## 3. Use the skill. + +Restart `seqera ai` so it discovers the new skill and loads it into the session context. +Your own skills do not appear as slash commands; the `/` palette is reserved for built-in, backend-provided skills. +Instead, CoScientist now follows your skill whenever you ask for a test: + +```text +Add an nf-test for the FASTQC process. +``` + +It applies the stable-versus-unstable rules from your skill automatically, without you restating them. +To make the skill available across all your projects rather than one, place the same directory under `~/.agents/skills/` instead. + +!!! note "Checkpoint" + + With the skill in place, asking for a test produces one that asserts on stable output and excludes unstable content, without you re-explaining the rules. + +### Takeaway + +You captured the testing discipline as a reusable skill that CoScientist loads as session context. +You and your teammates now get a consistent `nf-test` on demand, without re-explaining the rules each time. + +### What's next? + +[Wrap up the side quest](next_steps.md). diff --git a/docs/en/docs/side_quests/co_scientist/img/.gitkeep b/docs/en/docs/side_quests/co_scientist/img/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/en/docs/side_quests/co_scientist/img/chat-landing.png b/docs/en/docs/side_quests/co_scientist/img/chat-landing.png new file mode 100644 index 0000000000..c281b87193 Binary files /dev/null and b/docs/en/docs/side_quests/co_scientist/img/chat-landing.png differ diff --git a/docs/en/docs/side_quests/co_scientist/img/explain-with-ai.png b/docs/en/docs/side_quests/co_scientist/img/explain-with-ai.png new file mode 100644 index 0000000000..24ec7fc7ad Binary files /dev/null and b/docs/en/docs/side_quests/co_scientist/img/explain-with-ai.png differ diff --git a/docs/en/docs/side_quests/co_scientist/img/launchpad-rnaseq-nf.png b/docs/en/docs/side_quests/co_scientist/img/launchpad-rnaseq-nf.png new file mode 100644 index 0000000000..b5e7ab95d8 Binary files /dev/null and b/docs/en/docs/side_quests/co_scientist/img/launchpad-rnaseq-nf.png differ diff --git a/docs/en/docs/side_quests/co_scientist/index.md b/docs/en/docs/side_quests/co_scientist/index.md new file mode 100644 index 0000000000..f9da131db2 --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/index.md @@ -0,0 +1,45 @@ +--- +title: Building with Seqera AI +hide: + - toc +--- + +# Building with Seqera AI + +CoScientist is Seqera's AI assistant for bioinformatics, integrated directly into the Seqera Platform. +It can take on the repetitive parts of pipeline work, from finding modules and wiring processes to launching runs, debugging failures, and writing tests, so you spend less time on plumbing and more on the science. +This side quest drives it hands-on against the [`nextflow-io/rnaseq-nf`](https://github.com/nextflow-io/rnaseq-nf) demo pipeline, taking you from a first conversation through development, debugging, a pull request, a test, and your own reusable agent skill. + +## Audience & prerequisites + +This side quest is designed for learners who have completed the [Hello Nextflow](../../hello_nextflow/index.md) course and are comfortable working on the command line with basic Nextflow concepts. + +**Prerequisites** + +- Completed [Hello Nextflow](../../hello_nextflow/index.md) or equivalent. +- Comfortable with the CLI and familiar with basic Nextflow concepts. + +**Provided sandbox:** a training Platform workspace with CoScientist enabled, a working compute environment, and a GitHub account to fork into. + +## Learning objectives + +By the end of this side quest, you will be able to: + +- Converse with CoScientist in the web chat UI to explore and plan pipeline work. +- Understand how CoScientist takes real actions on the Seqera Platform and GitHub on your behalf. +- Register a pipeline on the Launchpad and inspect compute and data assets. +- Work effectively with the agent: prompt for intent, verify what it did, and redirect it when it goes wrong. +- Fix a failed run from the user side, through launch parameters, without changing code. +- Fork `rnaseq-nf`, add a process by reusing an nf-core module, and open a pull request, all from the web chat. +- Move to the `seqera ai` CLI to write an nf-test and run it to green. +- Add a GitHub Actions workflow that runs the tests on every change. +- Package the nf-test workflow into a reusable CoScientist skill and invoke it as a slash command. + +## The example pipeline + +[`nextflow-io/rnaseq-nf`](https://github.com/nextflow-io/rnaseq-nf) is a minimal RNA-seq pipeline that runs `INDEX` and `FASTQC` in parallel, followed by `QUANT` (Salmon) and `MULTIQC`. +It ships test data in `data/ggal/` and has no nf-test suite yet. That makes it a good target for adding one. + +**Time estimate:** ~3 hours + +[Get started :material-arrow-right:](01_meet_coscientist.md){ .md-button .md-button--primary } diff --git a/docs/en/docs/side_quests/co_scientist/next_steps.md b/docs/en/docs/side_quests/co_scientist/next_steps.md new file mode 100644 index 0000000000..a8482646f3 --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/next_steps.md @@ -0,0 +1,66 @@ +# Next Steps + +Congratulations on completing the **Building with Seqera AI** training course. + +As you put CoScientist to work on your own pipelines, keep doing what you practised here: let the agent move fast, and stay the reviewer who checks its output before relying on it. + +--- + +## 1. What else CoScientist can do. + +This side quest followed one end-to-end flow, but CoScientist does considerably more. + +- **Build, plan, and goal modes**: make changes directly, plan read-only before acting, or let goal mode keep working until an objective is met. +- **Built-in slash commands**: ready-made workflows in the CLI such as `/debug-last-run-on-seqera`, `/nextflow-config`, and `/nextflow-schema`. +- **Session resumption**: continue your most recent session from the CLI with `seqera ai -c`. +- **Command approval**: choose how much the agent runs automatically (`basic`, `default`, or `full`) so you stay in control of what executes. +- **Code intelligence**: language-server support for Nextflow, Python, and R that flags errors as code is written. +- **Script conversion**: convert R and Python scripts and Jupyter notebooks toward Nextflow. +- **Wave containers**: build containers from `conda` or `pip` packages without writing a Dockerfile. +- **nf-core modules**: discover and reuse from over 1,000 standardized nf-core modules. + +See the [CoScientist documentation](https://docs.seqera.io/platform-cloud/co-scientist/) for the full feature set. + +!!! info "Cloud and Enterprise" + + This training runs on Seqera Cloud, which uses Seqera's hosted model and meters usage through credits. + On Seqera Enterprise (currently AWS only), CoScientist can run Claude inference inside your own AWS account through Amazon Bedrock, so prompts and data stay within that account. + +--- + +## 2. Keep building with CoScientist. + +Here are three ways to continue developing with CoScientist. + +### 2.1. Apply CoScientist to your own pipelines. + +Try CoScientist against one of your own Nextflow pipelines or a real nf-core pipeline. +Working on a codebase you know well is the fastest way to see where CoScientist adds value and where you need to steer it. + +### 2.2. Use the CLI in your development environment. + +The `seqera ai` CLI brings the same agent into your local terminal. +Install it and run it alongside your usual development tools so CoScientist can read and edit files directly in your working directory. + +### 2.3. Read the documentation. + +The [CoScientist documentation](https://docs.seqera.io/platform-cloud/co-scientist/) covers capabilities, configuration, and usage patterns in detail. +Refer to it when you want to understand what the agent can and cannot do, or when you need to configure it for a specific environment. + +--- + +## 3. Get help from the community. + +- [Nextflow Slack](https://www.nextflow.io/slack-invite.html): Ask questions and share your work +- [Community forum](https://community.seqera.io/): Discuss ideas and get advice + +--- + +## 4. Continue your Nextflow training. + +If you haven't already, check out our other training courses: + +- **[Hello Nextflow](../../hello_nextflow/index.md)**: Foundational Nextflow concepts +- **[Testing with nf-test](../nf_test/index.md)**: Writing and running tests for workflows +- **[Troubleshooting Workflows](../debugging/index.md)**: Identifying and fixing common workflow errors +- **[Side Quests](../index.md)**: Deep dives into specific topics diff --git a/docs/en/docs/side_quests/co_scientist/solutions/pipeline.nf.test b/docs/en/docs/side_quests/co_scientist/solutions/pipeline.nf.test new file mode 100644 index 0000000000..5485def23d --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/solutions/pipeline.nf.test @@ -0,0 +1,27 @@ +// Illustrative pipeline-level nf-test for rnaseq-nf. +// +// This is a reference example for the side quest, not a validated test. +// It runs the whole pipeline on the bundled ggal_gut test data and checks the +// published outputs. Run `nf-test test` against your fork and adjust the output +// paths and snapshot to match what nf-test actually produces before relying on it. + +nextflow_pipeline { + + name "Test rnaseq-nf end to end" + script "main.nf" + + test("Default ggal_gut run") { + + then { + // The whole pipeline completes. + assert workflow.success + + // Stable: snapshot the Salmon quantification table. + // Do NOT snapshot multiqc_report.html, the FastQC zips, or the Salmon + // logs and cmd_info.json; those embed timestamps, versions, and paths. + assert snapshot( + path("${launchDir}/results/quant/ggal_gut/quant.sf") + ).match() + } + } +} diff --git a/docs/en/docs/side_quests/co_scientist/solutions/write-nf-test/SKILL.md b/docs/en/docs/side_quests/co_scientist/solutions/write-nf-test/SKILL.md new file mode 100644 index 0000000000..d737925081 --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/solutions/write-nf-test/SKILL.md @@ -0,0 +1,29 @@ +--- +name: write-nf-test +description: Generate an nf-test that asserts on stable output and excludes unstable content. +--- + +# Write nf-test + +When asked to write an nf-test for a process or a pipeline, follow the rules below. + +## Steps + +1. Scaffold the test under `tests/`. + For a process use a `nextflow_process` block; for a pipeline use a `nextflow_pipeline` block. + Provide representative inputs or test data so the test runs end to end. + +2. Assert on deterministic output only: + + - Numeric or tabular result files, such as a Salmon `quant.sf` table + - The existence of the expected output files and directories + +3. Do NOT snapshot unstable content: + + - Reports that embed timestamps or versions, such as MultiQC HTML + - Log files, and files that record the command line or tool version (for example Salmon `cmd_info.json` and `meta_info.json`) + - Any path or value containing the work-directory hash + - Version strings + +4. Run `nf-test test` on the generated file. + When it fails, narrow the assertion rather than snapshotting unstable content, and re-run until it passes. diff --git a/docs/en/docs/side_quests/co_scientist/survey.md b/docs/en/docs/side_quests/co_scientist/survey.md new file mode 100644 index 0000000000..cb8ba7731f --- /dev/null +++ b/docs/en/docs/side_quests/co_scientist/survey.md @@ -0,0 +1,7 @@ +# Feedback survey + +Before you move on, please complete this short 5-question survey to rate the training, share any feedback you may have about your experience, and let us know what else we could do to help you in your Nextflow journey. + +This should take you only a minute or two to complete. Thank you for helping us improve our training materials for everyone! + + diff --git a/docs/en/docs/side_quests/index.md b/docs/en/docs/side_quests/index.md index 37d109dc8e..573536998a 100644 --- a/docs/en/docs/side_quests/index.md +++ b/docs/en/docs/side_quests/index.md @@ -16,6 +16,7 @@ additional_information: - Work efficiently with files using Nextflow file operations - Debug common workflow issues systematically - Use and build Nextflow plugins + - Use Seqera AI (CoScientist) to develop, debug, and test pipelines on the Platform audience_prerequisites: - "**Audience:** This collection is designed for learners who have completed the Hello Nextflow beginner course and want to go deeper into specific topics." - "**Skills:** Experience with the command line and familiarity with basic Nextflow concepts and tooling is assumed." @@ -47,6 +48,7 @@ If this is your first time exploring the Side Quests, start with the [Orientatio | [Troubleshooting Workflows](./debugging/index.md) | Identifying and fixing common workflow errors | 1 hour | | [Workflows of Workflows](./workflows_of_workflows/index.md) | Composing complex pipelines from reusable named workflow modules | 30 mins | | [Plugin Development](./plugin_development/index.md) | Using and building Nextflow plugins | 3 hours | +| [Building with Seqera AI](./co_scientist/index.md) | Use Seqera AI (CoScientist) to develop, run, debug, and test pipelines | 3 hours | [Get started :material-arrow-right:](orientation.md){ .md-button .md-button--primary } diff --git a/docs/en/docs/side_quests/platform_automation/img/one-api.excalidraw.svg b/docs/en/docs/side_quests/platform_automation/img/one-api.excalidraw.svg new file mode 100644 index 0000000000..7a3f55c89c --- /dev/null +++ b/docs/en/docs/side_quests/platform_automation/img/one-api.excalidraw.svg @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + + + Web + + + + + + + CLI + + + + + + + + Terraform + + + + + + + + + API + + STANDARD INTERFACE + + + + + + + seqera + PLATFORM + + + + + + + + + + + + + + GitHub + + + + + + + Cloud + + AWS + + Azure + + GCP + + + + + + + + + + + + + Containers + diff --git a/docs/en/docs/side_quests/platform_automation/index.md b/docs/en/docs/side_quests/platform_automation/index.md new file mode 100644 index 0000000000..335bc40645 --- /dev/null +++ b/docs/en/docs/side_quests/platform_automation/index.md @@ -0,0 +1,712 @@ +# Seqera Platform Automation + +The Seqera Platform does not run your work. +It is an API and a control plane over your cloud. +It hands jobs to a compute environment, the compute environment runs them on cloud VMs, and the Platform reads back state, logs, and exit codes. + +Everything the web UI does, it does by calling the Platform API. So everything is automatable: with the API, Terraform, and the CLI you can manage compute environments, pipelines, and runs as code, with no clicking. This side quest walks that programmatic surface from the most privileged role to the least. + +
+--8<-- "docs/en/docs/side_quests/platform_automation/img/one-api.excalidraw.svg" +
+ +### Learning goals + +In this side quest, we'll drive the Platform across three workspace roles of decreasing permission: **Admin**, **Maintain**, and **Launch**. Each role does one job and hands an artifact to the next. You'll learn how to: + +- Create a compute environment two ways, with increasing control over the cloud: the UI (Batch Forge) and Terraform +- Add a pipeline to the Launchpad declaratively with Terraform, and see idempotency +- Launch a pipeline using the GUI, CLI and `seqerakit`. +- Tell declarative existence (Terraform) apart from imperative actions (the UI, `seqerakit`, `tw`) and learn when to use each + +### Prerequisites + +Before taking on this side quest, you should: + +- Have a Seqera Platform account on Seqera Cloud [https://cloud.seqera.io](`https://cloud.seqera.io`), or an Enterprise install +- Be a member of a workspace with a role: Admin, Maintain, or Launch +- Have credentials for your cloud provider already added to that workspace (not covered here, see [credentials overview](https://docs.seqera.io/platform-cloud/credentials/overview)) +- Ideally, GitHub credentials are added to your workspace. This is optional, but it would help prevent incidents caused by GitHub rate limits. +- Be comfortable with the command line and basic Nextflow concepts + +### Know your role + +The work is split by role, from most to least privileged. Start at the highest tier your access allows: + +| Tier | Role | Can change | Produces | +| ------------------------------------------------------- | ------------- | ---------------------------------------- | --------------------- | +| [Admin](#1-admin-compute-environments-and-cloud) | Owner / Admin | Compute environments and cloud resources | a Compute environment | +| [Maintain](#2-maintain-add-a-pipeline-to-the-launchpad) | Maintain | Pipelines on the Launchpad | a Launchpad pipeline | +| [Launch](#3-launch-run-a-pipeline) | Launch | Nothing; can only run | pipeline runs | + +An Admin user will learn how to create compute environments. If you only have Maintain, your workspace must already include one valid compute environment; start at section 2. If you only have Launch, your workspace must have a pre-configured pipeline (and optionally an Action) to run; start at section 3. Each section opens with what it requires. + +--- + +## 0. Get started + +### Open the training codespace + +For this tutorial you will need the following tools: + +- [`tw`](https://github.com/seqeralabs/tower-cli/) +- [`seqerakit`](https://github.com/seqeralabs/seqerakit) +- [Terraform](https://www.terraform.io/) + +The Codespace should already contain all of these tools, but you can verify them by trying their respective CLI commands. + +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/nextflow-io/training?quickstart=1&ref=master) + +### Move into the project directory + +The Codespace terminal opens at the repository root (`/workspaces/training`). All the assets for this side quest live under `side-quests/platform_automation/`, so move there now: + +```bash +cd side-quests/platform_automation +``` + +Focus VSCode on this directory so the file explorer shows the assets you'll edit: + +```bash +code . +``` + +### Review the materials + +The directory holds the Terraform and `seqerakit` configurations for each role. Each section below tells you which subdirectory to `cd` into: + +??? abstract "Directory contents" + + ```console + . + ├── terraform + │ ├── compute-env # section 1 (Admin): provision the cloud + compute environment + │ │ ├── main.tf + │ │ ├── variables.tf + │ │ └── terraform.tfvars.example + │ └── pipeline # section 2 (Maintain): add a pipeline to the Launchpad + │ ├── main.tf + │ ├── variables.tf + │ └── terraform.tfvars.example + └── seqerakit # section 3 (Launch): launch a pipeline + ├── launch-rnaseq.yml + └── launch-rnaseq-multiple.yml + ``` + +We'll cover the content of these files in a moment. + +### Check workspace access + +This training requires you to be in a workspace. Navigate to [Seqera Platform](https://cloud.seqera.io/) or your Enterprise install and check you can see the relevant workspace which should be under an organization. There should already be credentials pre-configured and if you have Maintain or lower permissions, a compute environment and/or pipeline should already exist. If you cannot see the workspace, please ask your workspace owner to add you to one. + +### Create an access token + +We authenticate to the Platform to tell it who we are. That is what grants the permissions of our role. Seqera uses a **personal access token**: a bearer token you create once and send with every API call, in an `Authorization: Bearer ` header. The token carries your identity and role, so the Platform knows who you are and what you can do. Anyone with the token can act as you, so keep it secret. In this training we save it as an environment variable. + +1. In the Platform, open the user menu (top right) and choose **Your tokens**. +2. Click **Add token**, name it (e.g. `platform-automation`), and click **Add**. +3. Copy the token now. The Platform shows it only once. +4. In the Codespace terminal, export it: + +```bash +export TOWER_ACCESS_TOKEN= +``` + +Terraform, `tw`, and `seqerakit` all read the token from this variable. Check it worked: + +```bash +tw info +``` + +This prints the API endpoint and the authenticated user. If it errors, the token is wrong or not exported. + +!!! warning + + This only exists in a single terminal session, if you open a new terminal you will need to export them again. + +For Seqera Enterprise, also set the environment variable `TOWER_API_ENDPOINT` to your install's API URL. Everything else is identical. + +```bash +export TOWER_API_ENDPOINT= +``` + +In Terraform, you must set the `server_url` variable to the same value. + +```hcl +provider "seqera" { + server_url = "" +} +``` + +### Find your organization and workspace IDs + +Later steps need the **numeric workspace ID**. The web UI navigates by name, so the number is not in the URL; find it like this: + +1. In the Platform, click your organization name to open the organization page. +2. Open the **Workspaces** tab. Each workspace is listed with its numeric ID. That + number is the `workspace_id` the Terraform and API steps ask for. +3. You only need the numeric organization ID if you list workspaces over the API; if you read the workspace ID from the Workspaces tab, you can skip it. + +Or list everything your token can reach from the terminal: + +```bash +tw workspaces list +``` + +This prints a table with the workspace ID, the workspace name, and the organization name. + +Two forms of the same workspace turn up in this module. Terraform and the API want the **numeric** ID (`workspace_id`). `tw` and `seqerakit` accept either the numeric ID or the `Organization/Workspace` **name** (e.g. `my-org/platform-automation`). Export the numeric ID once so `tw` targets the shared workspace without a `--workspace` flag on every command: + +```bash +export TOWER_WORKSPACE_ID= +``` + +#### Pick a workshop handle + +In a shared workspace, every pipeline you add needs a name no one else is using. Pick a short handle and export it once; everything from section 2 onward reuses this one variable. Use only letters, numbers, and hyphens. Launchpad names cannot contain dots or spaces: + +```bash +export WORKSHOP_USER= # e.g. ada, jdoe, team-rocket +``` + +We set this by hand rather than reading `$USER`: in the Codespace everyone shares the same Linux user, so `$USER` is identical for all learners and would collide. + +--- + +## 1. Admin: compute environments and cloud + +**Requires:** Owner or Admin role, and permissions in the cloud environment. If you are using a Cloud provider, authenticate with them first via their CLI (e.g. `aws sso login`). There must be valid credentials in the workspace to perform the operations in the Cloud and build compute environments, please check the Seqera documentation for further details. + +The same compute environment can be created with increasing control over the cloud. +You'll build it two ways. + +!!! note "This quest uses Azure Batch" + + Section 1.1 covers AWS, Azure, GCP, on-prem HPC, and Kubernetes, but everything after it (the `tw` export, the Terraform, and the `az://work` work directory used throughout) assumes an Azure Batch compute environment. + Users on other providers can follow the same pattern, but the provider-specific resources and work-directory paths will differ. + +### 1.1. Create a compute environment in the UI + +On the cloud Batch providers (AWS, Azure, GCP), the UI offers **Batch Forge**: the hands-off option where the Platform reaches into your cloud and creates the resources for you. Most convenient, least control. On-prem HPC clusters and Kubernetes use a different, non-Forge flow: the cluster already exists and you point the Platform at it. + +In the side bar, open **Compute Environments** and click **Create compute environment**. Give it a name, pick your platform, and select the credentials for it. The rest of the form differs by provider. Select yours below for recommended settings: + +=== "AWS" + + - Region + - Work directory (S3 bucket) + - Config mode should be Batch Forge + - Provisioning model should be Spot, using Spot instances for nodes which run Nextflow tasks. + +=== "Azure" + + - Location + - Work directory (Azure Storage container) + - Select "Separate head and worker pools" to create one pool for the Nextflow head job and a separate pool for the tasks. + - For the head pool VM type, select a VM you have sufficient quota for. Standard_D2s_v3 (2 CPUs) is a good starting choice. + - Set the head pool VM count to 1. The head pool runs a single Nextflow head job; it does not need more. + - For the worker pool, select a VM type you have sufficient quota for. Standard_D4s_v3 (4 CPUs) is a good starting choice. + - Set the worker pool VM count to 4 + - Leave autoscaling enabled for both pools + +=== "GCP" + + - Location (region) + - Work directory (Cloud Storage bucket, e.g. `gs://my-bucket`) + - Enable Spot instances under GCP Resources to reduce cost. + - Config mode should be Batch Forge + +=== "On-prem / HPC" + + This is **not** Batch Forge: no pools are created for you. The cluster already exists and the Platform submits to it over SSH or the Tower Agent. + + - Compute platform: your scheduler (Slurm, LSF, PBS Pro, Grid Engine, Moab) + - Credentials (SSH key or Tower Agent) + - Login hostname: the cluster's login node + - Work directory: an absolute path on the shared filesystem + - Launch directory (defaults to the work directory if omitted) + - Head queue name: the queue the Nextflow head job submits to + - Compute queue name: the queue Nextflow submits tasks to + +=== "Kubernetes" + + Also **not** Batch Forge: a manual flow against a cluster you prepare first (apply a YAML manifest creating the service accounts and role bindings). + + - Credentials (service-account token or X509 client certificate) + - Control-plane URL (from `kubectl cluster-info`) + - SSL certificate (from `~/.kube/config`) + - Namespace (default `tower-nf`) + - Head service account (default `tower-launcher-sa`) + - Storage claim name (default `tower-scratch`) + - Storage mount path (default `/scratch`) + - Work directory: the storage mount path or a subdirectory of it + - Head job CPUs and memory + +On the cloud Batch providers (AWS, Azure, GCP), leave the other settings at their defaults, keep **Dispose resources** enabled so the pool is torn down when you delete the compute environment, and click **Add** in the top right. + +Watch your cloud console and you will see Forge create the resources: an identity and roles for Nextflow (access to blob storage and the Batch service), a more limited role for the worker tasks (storage only), the pools, and their networking. Many moving parts, all handled for you. The on-prem and Kubernetes paths have nothing to forge; they submit to infrastructure you already run. + +That convenience is the trade-off: Forge owns those resources and gives you little control. A lab that already runs on managed infrastructure wants the opposite, to describe the resources itself and wire the Platform to them. That is the Terraform path. + +### 1.2. Read it back over the API with `tw` + +The form you just filled in did nothing special: it sent a configuration to the Platform API. +Ask the API for what you made, from the terminal, with nothing retyped. + +`tw` reads the same workspace the UI does. List the compute environments and the one you created in 1.1 is there: + +```bash +tw compute-envs list +``` + +Now export its full configuration to a file. +Use the name you gave it in 1.1: + +```bash +tw compute-envs export forge-ce.json --name="" +``` + +`tw` reads the workspace from `TOWER_WORKSPACE_ID` (set in section 0), so no `--workspace` flag is needed. +The file `forge-ce.json` is the compute environment as the API stores it: + +```json title="forge-ce.json" +{ + "discriminator": "azure-batch", + "region": "eastus", + "workDir": "az://work", + "waveEnabled": true, + "fusion2Enabled": true, + "forge": { + "headPool": { + "vmType": "Standard_D2s_v3", + "vmCount": 1, + "autoScale": true + }, + "workerPool": { + "vmType": "Standard_D4s_v3", + "vmCount": 4, + "autoScale": true + }, + "disposeOnDeletion": true, + "dualPoolConfig": true + } +} +``` + +The `forge` block is exactly what the checkboxes set in 1.1. Its presence is what tells the Platform to create and scale the pools for you. In the next section, the absence of this block is what makes the Terraform compute environment manual. + +This file can be used to recreate the compute environment, with nothing retyped. + +```bash +tw compute-envs import forge-ce.json --name="azure-batch-copy" +``` + +The UI sent this JSON to create the compute environment; `tw export` reads it back, and `tw import` sends it again. The GUI and the CLI are two clients of the same API. + +The exported JSON does not include a credentials ID. If your workspace has more than one credential for this cloud, `tw` cannot tell which to use and fails with `Multiple credentials match this compute environment`. Name one explicitly with `-c` (see `tw credentials list`): + +```bash +tw compute-envs import forge-ce.json --name="azure-batch-copy" --credentials="" +``` + +### 1.3. Provision the cloud with Terraform + +Terraform manages cloud resources declaratively: you describe what should exist and Terraform makes it so. It does not run your work; it only creates the compute environment. `side-quests/platform_automation/terraform/compute-env/` does it in a single `main.tf`, two providers in one apply: + +- data sources: the existing Batch account and the Azure credentials already + stored in the Platform, referenced, not created. +- `azurerm`: creates a head pool and a worker pool on that account. +- `seqera`: creates the compute environment pointing at those pools. + +The manual marker: `head_pool` is set to the pool Terraform just made and there is **no `forge` block**. That one difference is what makes the compute environment manual instead of Forge. `nextflow_config` routes tasks to the worker pool. + +The whole file is in the repo to read at your own pace, and most of it is standard Terraform: the providers, the locals, and the two Azure Batch pools. +Only one block encodes the lesson that makes this compute environment _manual_, so we'll focus there and treat the rest as reference. + +The first block declares the providers, pins their versions and sets them up. The `seqera` provider is pinned to exactly `0.40.1`: + +```terraform title="terraform/compute-env/main.tf" hl_lines="5" +terraform { + required_version = ">= 1.9" + + required_providers { + seqera = { source = "seqeralabs/seqera", version = "0.40.1" } + azurerm = { source = "hashicorp/azurerm", version = ">= 3.0" } + random = { source = "hashicorp/random", version = ">= 3.0" } + } +} + +provider "azurerm" { + features {} + subscription_id = var.subscription_id +} + +provider "seqera" { + server_url = var.server_url +} +``` + +Let's move into the compute env directory and initialise the Terraform provider: + +```bash +cd terraform/compute-env +terraform init +``` + +You should see Terraform install the relevant providers and create a lockfile. This makes sure anyone using this Terraform is using the same provider versions. Terraform can and should be shared, so it is very important to make sure everyone uses the same versions! + +Next we can define some variables we can use throughout our deployment. In this case, we will set the node sizes and details once and reuse them throughout the configuration: + +```terraform title="terraform/compute-env/main.tf" hl_lines="2 3" +locals { + head_vm_size = "Standard_E4ds_v5" + worker_vm_size = "Standard_E8ds_v5" + worker_max_nodes = 8 + node_agent_sku_id = "batch.node.ubuntu 24.04" + + # The Platform requires a pool's task slots to equal the node's vCPU count. + # Azure VM size names embed that count (Standard_E8ds_v5 -> 8), so derive it + # from the size rather than hardcoding a number that can drift out of sync. + head_vm_cores = tonumber(regex("^Standard_[A-Z]+([0-9]+)", local.head_vm_size)[0]) + worker_vm_cores = tonumber(regex("^Standard_[A-Z]+([0-9]+)", local.worker_vm_size)[0]) +} +``` + +Now the resources Terraform does create. In this code, Terraform creates two pools in Azure Batch with the `azurerm_batch_pool` resource. You can add as many as you like, but the details must match the values the Azure provider expects. We won't go into the details of the resource here, you can check them yourself, but you can see how some details refer to variables and outputs of other steps: + +```terraform title="terraform/compute-env/main.tf" +# Head pool: runs the Nextflow head job. One node is enough. +resource "azurerm_batch_pool" "head" { + name = "rnaseq-head-${random_string.suffix.result}" + display_name = "rnaseq head pool" + resource_group_name = var.batch_account_rg + account_name = var.batch_account_name + vm_size = local.head_vm_size + node_agent_sku_id = local.node_agent_sku_id + max_tasks_per_node = local.head_vm_cores + # Etc... +``` + +Once we've built the Azure Batch node pools, we add the Seqera Platform compute environment that points to them. This is the manual marker in code: `head_pool` points at the pool we just created and `nextflow_config` routes tasks to the worker pool's queue. Because they reference `azurerm_batch_pool.head.name` and `azurerm_batch_pool.worker.name` respectively, Terraform knows to create the pools on Azure Batch first: + +```terraform title="terraform/compute-env/main.tf" hl_lines="14 17" +resource "seqera_compute_env" "main" { + workspace_id = var.workspace_id + + compute_env = { + name = "azure-batch-manual" + description = "Azure Batch CE on Terraform-managed pools" + platform = "azure-batch" + credentials_id = local.azure_credentials_id + + config = { + azure_batch = { + region = var.region + work_dir = var.work_dir + head_pool = azurerm_batch_pool.head.name + worker_pool = azurerm_batch_pool.worker.name + enable_wave = true + enable_fusion = true + } + } + } +} +``` + +Now, we can run Terraform to apply these changes, where it will create our Azure Batch pools and then the Seqera Platform compute environment. + +```bash +terraform apply +``` + +After you run `terraform apply`, it will ask you to fill in the details defined in `variables.tf` + +- `subscription_id` is the subscription of your Azure account, which you can find in the Azure Portal under **Subscriptions**. +- `workspace_id` is the numeric workspace ID you exported in section 0. +- `region` is the Azure region you want to deploy to, e.g. `eastus`. +- `work_dir` is the Azure Blob work directory you want to use, e.g. `az://work`. +- `batch_account_name` is the name of the Azure Batch account you want to use, which you can find in the Azure Portal under **Batch accounts**. +- `batch_account_rg` is the resource group of the Azure Batch account you want to use, which you can find in the Azure Portal under **Batch accounts**. +- `azure_credentials_name` is the ID of the Azure credentials you want to use, which you can find in the Platform under **Credentials**. +- `server_url` is the URL of your Seqera Platform instance, which you can find in the Platform under **Settings**. This will default to the cloud instance of Seqera Platform by default. + +!!! tip "Re-using variables" + + You can save them as a .tfvars file and Terraform will read them automatically, so you don't have to type them each time. See `terraform.tfvars.example` for the full variable reference. + Alternatively, you can save them as an environment variable preceded by `TF_VAR_`, for example `TF_VAR_subscription_id=00000000-0000-0000-0000-000000000000`. + +Terraform will show you a preview, if it looks accurate, type `yes` to apply the changes. It will create the pools and the compute environment, and print the `compute_env_id` you hand to the Maintain tier. + +### 1.4. See both sides + +The compute environment now exists in two places, and as an Admin with cloud access you can see both. + +On the Platform side, read the ID Terraform produced: + +```bash +terraform output compute_env_id +``` + +If you have access to the Azure side, open the Batch account in the portal. You will see the head and worker pools Terraform created, sitting idle with no jobs yet. Once you launch a pipeline (sections 2 and 3), jobs and tasks stack onto these pools, and you can drill into a task's logs and exit code. That is the whole point of the Admin tier: to create cloud resources for other users to work on. Maintain- and Launch-role users see only the compute environment in the workspace, not the cloud behind it. + +### Takeaway + +One compute environment, three levels of control: Forge in the UI (easiest, the Platform owns the pool), `tw` (export and import the same config over the API), and Terraform (you own the pools and the wiring, end to end). The artifact you hand to the Maintain tier is a `compute_env_id`. + +!!! note "Tearing it down" + + Terraform manages the cloud resources it created, so it can remove them too. From `terraform/compute-env`, run `terraform apply -destroy` to delete the compute environment and the Batch pools in one step. The existing Batch account and credentials are referenced, not managed, so they are left untouched. + +--- + +## 2. Maintain: add a pipeline to the Launchpad + +**Requires:** Maintain role, and a `compute_env_id` (from section 1 or an Admin on your team) plus the numeric `workspace_id`. Maintain manages pipelines, not compute environments. That split is deliberate: Admin owns the compute environment, you own your pipelines. + +You add the same pipeline, `rnaseq-nf-$WORKSHOP_USER`, two ways. The first, `tw`, is imperative: you run a command and it acts. The second, Terraform, is declarative. Watch what happens when you run each one twice. + +### 2.1. Add a pipeline via `tw` + +The CLI adds a pipeline in one command. + +!!! note + + Set the compute-env name to the one you created in section 1.4, or the one your Admin provided. We've left it as `azure-batch-manual` to match the Terraform example above, assuming you won't delete the compute environment in between. + +```bash +tw pipelines add \ + --name="rnaseq-nf-$WORKSHOP_USER" \ + --compute-env="azure-batch-manual" \ + --work-dir="az://work" \ + --revision="master" \ + https://github.com/nextflow-io/rnaseq-nf +``` + +Run it again and `tw` errors: a pipeline with that name already exists. The command is imperative, so each invocation tries to add a pipeline; it has no notion of "already in the desired state". To make "add once, and leave it alone after that" the _default_ behaviour, we need a tool that manages existence rather than actions: Terraform. + +The next section adds a pipeline with the same name using Terraform, so delete this one first to start from a clean slate: + +```bash +tw pipelines delete --name="rnaseq-nf-$WORKSHOP_USER" +``` + +### 2.2. Add a pipeline with Terraform + +`side-quests/platform_automation/terraform/pipeline` adds the same pipeline declaratively. You describe the pipeline that should exist; Terraform makes the workspace match. Like the imperative `tw` command above, the config tracks the `master` branch; you will pin it to a release tag below to see Terraform update the pipeline in place. + +```bash +cd /workspaces/training/side-quests/platform_automation/terraform/pipeline +terraform init +terraform apply -var="username=$WORKSHOP_USER" -var="workspace_id=" -var="compute_env_id=" +``` + +!!! tip + + Copy `terraform.tfvars.example` to `terraform.tfvars` and fill it in so you stop passing `-var` flags. + +Confirm it landed, in the UI (open the Launchpad) or from the terminal, the same way you read the compute environment back in section 1.2: + +```bash +tw pipelines list +``` + +`rnaseq-nf-$WORKSHOP_USER` is there, with no run. Now apply again: + +```bash +terraform apply -var="username=$WORKSHOP_USER" -var="workspace_id=" -var="compute_env_id=" +``` + +??? success "Command output" + + ```console + No changes. Your infrastructure matches the configuration. + ``` + +That is the difference. `tw` errored on the second run because it performs an action every time. Terraform manages whether the pipeline **exists**: it is already there, so there is nothing to do. Apply ten times, still one pipeline. This is what keeps you from accumulating competing, half-duplicated resources in a shared workspace. To remove the pipeline, `terraform apply -destroy` with the same vars. + +**Update in place.** Terraform does more than create-or-skip; it reconciles the pipeline to whatever the config says. The config tracks the `master` branch, which moves as the pipeline gets new commits. Pin it to a fixed release tag instead, so every launch runs the same code. Edit one line: in `main.tf`, line 35 of the `launch` block, change the revision from the `master` branch to the `v2.4` release tag. + +=== "After" + + ```hcl title="terraform/pipeline/main.tf" hl_lines="4" linenums="32" + launch = { + pipeline = "https://github.com/nextflow-io/rnaseq-nf" + # The master branch moves; pin a release tag (e.g. v2.4) for reproducible launches. + revision = "v2.4" + compute_env_id = var.compute_env_id + work_dir = var.work_dir + } + ``` + +=== "Before" + + ```hcl title="terraform/pipeline/main.tf" hl_lines="4" linenums="32" + launch = { + pipeline = "https://github.com/nextflow-io/rnaseq-nf" + # The master branch moves; pin a release tag (e.g. v2.4) for reproducible launches. + revision = "master" + compute_env_id = var.compute_env_id + work_dir = var.work_dir + } + ``` + +Then apply: + +??? success "Command output" + + ```console + Terraform will perform the following actions: + + # seqera_pipeline.rnaseq_nf will be updated in-place + ~ resource "seqera_pipeline" "rnaseq_nf" { + ~ launch = { + ~ revision = "master" -> "v2.4" + # (4 unchanged attributes hidden) + } + } + + Plan: 0 to add, 1 to change, 0 to destroy. + ``` + +One pipeline, modified; not a second pipeline, and not an error. That is the third declarative behaviour, alongside create and no-op: update to match. Moving from a branch to a release tag is exactly the kind of change you want under version control: the pinned revision is now recorded in `main.tf`. + +!!! note "Pinning a revision vs. Launchpad versions" + + The Platform also has a separate concept of **Launchpad pipeline versions**: several saved launch configurations on one pipeline, with one marked as the default (most recent). You can use this to maintain and control pipeline versions within one concept of a "pipeline". + + The Terraform provider can also publish and promote those versions, and flag drift if someone changes the default in the UI. See the provider's [pipeline versioning guide](https://github.com/seqeralabs/terraform-provider-seqera/blob/master/docs/guides/pipeline-versioning.md). + +### Takeaway + +Two ways to add one pipeline, two mental models. `tw` is imperative: each run is an action, and adding twice is an error. Terraform is declarative: it manages existence, so a second apply is a no-op or an update-in-place. The artifact you hand to the Launch tier is the Launchpad pipeline `rnaseq-nf-$WORKSHOP_USER`. + +--- + +## 3. Launch: run a pipeline + +**Requires:** Launch role and a token with that role, plus a pipeline on the Launchpad (added by a Maintainer in section 2). Launch can only run things; it cannot create or modify compute environments, pipelines, or Actions. + +Everything in this section runs the pipeline the Maintainer already configured. None of it changes the pipeline; launching is an action, not a state. + +### 3.1. Use the GUI + +Open the **Launchpad**, select `rnaseq-nf-$WORKSHOP_USER`, and click **Launch**. The compute environment, revision, and work directory are already filled in by the Maintainer, so a Launch user just clicks the button. Submit it and the run appears under **Runs**. + +### 3.2. Use the `tw` CLI + +```bash +tw launch rnaseq-nf-$WORKSHOP_USER +``` + +It should show something like: + +??? success "Command output" + + ```console + Workflow 2ZXaU1AzEn7Onk submitted at [Organization / Workspace] workspace. + + https://cloud.seqera.io/orgs/Organization/workspaces/Workspace/watch/2ZXaU1AzEn7Onk + ``` + +You can monitor the run by clicking the provided URL. + +!!! note "Using a params file" + + If you wish to provide parameters to the pipeline, you can do so with the `--params-file` flag. + +### 3.3. Use `seqerakit` + +`seqerakit` is a command-line tool that wraps `tw`, driving it from YAML instead of flags. +Each YAML file declares the resources or actions to perform: here a `launch:` block lists the pipeline, workspace, and compute environment to run, with `${...}` values pulled from the environment. + +`seqerakit` launches a pipeline that already exists on the Launchpad, filling in the `tw launch` command from `seqerakit/launch-rnaseq.yml`. `WORKSHOP_USER` is set from section 0; set the compute environment name, dry run first, then launch: + +```bash +cd /workspaces/training/side-quests/platform_automation/seqerakit +export COMPUTE_ENVIRONMENT= + +seqerakit launch-rnaseq.yml --dryrun # prints the tw command, changes nothing +seqerakit launch-rnaseq.yml # launches for real +``` + +The dry run shows the underlying `tw` command. Run it twice and you get two runs. That is the imperative model: do the thing, now. + +The file you just ran is a single `launch` block: + +```yaml title="seqerakit/launch-rnaseq.yml" +launch: + - name: "rnaseq-nf-${WORKSHOP_USER}-run" + workspace: "${TOWER_WORKSPACE_ID}" + pipeline: "rnaseq-nf-${WORKSHOP_USER}" + compute-env: "${COMPUTE_ENVIRONMENT}" +``` + +One advantage of `seqerakit` is that the YAML forms a template of actions to perform. +This buys you two things. +First, you can save it and re-use it later. +Second, you can launch the same pipeline several times by adding more launch blocks to the YAML file, which is useful for launching with different parameters or on different compute environments: + +```yaml title="seqerakit/launch-rnaseq-multiple.yml" +launch: + - name: "rnaseq-nf-${WORKSHOP_USER}-run-1" + workspace: "${TOWER_WORKSPACE_ID}" + pipeline: "rnaseq-nf-${WORKSHOP_USER}" + compute-env: "${COMPUTE_ENVIRONMENT}" + - name: "rnaseq-nf-${WORKSHOP_USER}-run-2" + workspace: "${TOWER_WORKSPACE_ID}" + pipeline: "rnaseq-nf-${WORKSHOP_USER}" + compute-env: "${COMPUTE_ENVIRONMENT}" +``` + +Launch both runs now with: + +```bash +seqerakit launch-rnaseq-multiple.yml +``` + +### 3.4. Side note: Examine the cloud resources + +It's worth taking a second to examine the cloud resources here. + +A run does not create one neatly named cloud job. The platform submits a single job to the compute environment who's only job is to configure and run Nextflow. The Nextflow running within this job submits **many** Batch jobs and tasks, one per process invocation. Overall, this means you will not find a single resource named after the Platform run. Open the run on the Platform (**Runs** → your run), which mirrors the underlying Batch service: the task table here is the same work you can see in the cloud console. + +If you have cloud access, the prefixes Nextflow uses in each Batch service are: + +- **Azure Batch**: jobs named `job--`, tasks named `nf-`. +- **AWS Batch**: job names of the form `_`, with unsupported characters stripped (max 128 chars). +- **GCP Batch**: job IDs of the form `nf--`. + +The relationship is the point: the Platform hands work to Batch, Batch runs it on the pool VMs, and the Platform reads back state, logs, and exit codes. + +### 3.5. Side note: launch Actions + +There is one more way to launch, built for automation rather than people. An **Action** is a saved launch configuration behind a single URL: call that URL and the pipeline runs, with no Launchpad and no `tw`. A Maintainer creates one in the UI under **Actions** → **Add action**, picks the pipeline, compute environment and other settings, and saves it as a configuration. The Platform then shows a ready-made `curl` command for the Action's endpoint. + +The split mirrors the roles: _creating_ an Action needs the Maintain role, but _triggering_ it needs only a Launch token. That is the point of the stratified launch: a Maintainer hands launchers (or an automation system) a URL they can call to run the pipeline, and nothing more. + +### Takeaway + +Several ways to launch, all imperative: the GUI button, `tw launch`, `seqerakit`, and a launch Action's URL. Each does the thing, now; run it twice and you get two runs. That is the opposite of Terraform, which manages whether something exists. + +--- + +## Summary + +You drove the Seqera Platform programmatically across three roles, each handing an artifact to the next: Admin builds the compute environment and owns the cloud, Maintain adds pipelines and Launch triggers a run. + +### Key patterns + +- **Everything is one API.** The GUI, Terraform, `tw`, and `seqerakit` all call the same Platform API. Anything you can click, you can automate. +- **Use the right tool.** Terraform manages resources as state: what should exist, where. `tw` and `seqerakit` act on the Platform imperatively: they do the thing, now. + +| | Terraform | `seqerakit` / `tw` / Action | +| ------------ | ------------- | --------------------------- | +| Manages | existence | actions | +| Run twice | no-op | two runs | +| Mental model | desired state | do the thing, now | + +- **Roles stratify what each token can do.** A Maintain token defines pipelines and Actions; a Launch token can only trigger an Action and nothing else. The Action is the safe handoff from maintainers to launchers (and to automation). + +## What's next? + +- [Building with Seqera AI (CoScientist)](../co_scientist/index.md) side quest uses the same `rnaseq-nf` pipeline and API endpoints, but drives them via AI agents instead of manually. diff --git a/docs/en/mkdocs.yml b/docs/en/mkdocs.yml index 41ab59ce40..bbda02fb68 100644 --- a/docs/en/mkdocs.yml +++ b/docs/en/mkdocs.yml @@ -72,6 +72,15 @@ nav: - side_quests/nf_test/index.md - side_quests/debugging/index.md - side_quests/workflows_of_workflows/index.md + - Building with Seqera AI: + - side_quests/co_scientist/index.md + - side_quests/co_scientist/01_meet_coscientist.md + - side_quests/co_scientist/02_working_with_the_agent.md + - side_quests/co_scientist/03_develop_with_coscientist.md + - side_quests/co_scientist/04_move_to_the_cli.md + - side_quests/co_scientist/05_build_a_skill.md + - side_quests/co_scientist/survey.md + - side_quests/co_scientist/next_steps.md - Plugin Development: - side_quests/plugin_development/index.md - side_quests/plugin_development/01_plugin_basics.md diff --git a/side-quests/platform_automation/seqerakit/launch-rnaseq-multiple.yml b/side-quests/platform_automation/seqerakit/launch-rnaseq-multiple.yml new file mode 100644 index 0000000000..20c4928a03 --- /dev/null +++ b/side-quests/platform_automation/seqerakit/launch-rnaseq-multiple.yml @@ -0,0 +1,24 @@ +# Launch rnaseq-nf-$WORKSHOP_USER against the shared compute environment. +# +# This LAUNCHES a pipeline that already exists on the Launchpad. Terraform +# added it (see ../terraform/pipeline). seqerakit does not create it here. +# +# Declarative vs imperative: +# Terraform manages existence. apply twice = no second pipeline. +# seqerakit/tw do the thing, now. Run this twice = two runs. That is the +# point. Launching is an action, not a state. +# +# A bare `pipeline:` name (not a git URL) means "the saved Launchpad pipeline +# with this name". seqerakit expands ${WORKSHOP_USER} from the environment; +# export it first (see section 0). +# +# Set the workspace and compute-env names to match your workshop workspace. +launch: + - name: "rnaseq-nf-${WORKSHOP_USER}-run-1" + workspace: "${TOWER_WORKSPACE_ID}" + pipeline: "rnaseq-nf-${WORKSHOP_USER}" + compute-env: "${COMPUTE_ENVIRONMENT}" + - name: "rnaseq-nf-${WORKSHOP_USER}-run-2" + workspace: "${TOWER_WORKSPACE_ID}" + pipeline: "rnaseq-nf-${WORKSHOP_USER}" + compute-env: "${COMPUTE_ENVIRONMENT}" diff --git a/side-quests/platform_automation/seqerakit/launch-rnaseq.yml b/side-quests/platform_automation/seqerakit/launch-rnaseq.yml new file mode 100644 index 0000000000..9253657b1e --- /dev/null +++ b/side-quests/platform_automation/seqerakit/launch-rnaseq.yml @@ -0,0 +1,20 @@ +# Launch rnaseq-nf-$WORKSHOP_USER against the shared compute environment. +# +# This LAUNCHES a pipeline that already exists on the Launchpad. Terraform +# added it (see ../terraform/pipeline). seqerakit does not create it here. +# +# Declarative vs imperative: +# Terraform manages existence. apply twice = no second pipeline. +# seqerakit/tw do the thing, now. Run this twice = two runs. That is the +# point. Launching is an action, not a state. +# +# A bare `pipeline:` name (not a git URL) means "the saved Launchpad pipeline +# with this name". seqerakit expands ${WORKSHOP_USER} from the environment; +# export it first (see section 0). +# +# Set the workspace and compute-env names to match your workshop workspace. +launch: + - name: "rnaseq-nf-${WORKSHOP_USER}-run" + workspace: "${TOWER_WORKSPACE_ID}" + pipeline: "rnaseq-nf-${WORKSHOP_USER}" + compute-env: "${COMPUTE_ENVIRONMENT}" diff --git a/side-quests/platform_automation/terraform/compute-env/main.tf b/side-quests/platform_automation/terraform/compute-env/main.tf new file mode 100644 index 0000000000..ee00b4d61b --- /dev/null +++ b/side-quests/platform_automation/terraform/compute-env/main.tf @@ -0,0 +1,213 @@ +# Admin tier: Terraform creates the Azure Batch pools, then wires a manual +# Seqera compute environment to them. Manual = head_pool is set and there is no +# forge block, so the Platform submits to these pools but never creates or +# scales them. One config, two providers, end to end. +# +# Auth: run `az login`, then export TOWER_ACCESS_TOKEN (the seqera provider +# reads it from the environment). Sensitive keys come from terraform.tfvars or +# TF_VAR_ env vars; never commit them. + +terraform { + required_version = ">= 1.9" + + required_providers { + seqera = { source = "seqeralabs/seqera", version = "0.40.1" } + azurerm = { source = "hashicorp/azurerm", version = ">= 4.0" } + random = { source = "hashicorp/random", version = ">= 3.0" } + } +} + +provider "azurerm" { + features {} + subscription_id = var.subscription_id + resource_provider_registrations = "none" +} + +provider "seqera" { + server_url = var.server_url +} + +# Pool sizing and image. Edit here rather than exposing every value as a knob. +locals { + head_vm_size = "Standard_E4ds_v5" + worker_vm_size = "Standard_E8ds_v5" + worker_max_nodes = 8 + node_agent_sku_id = "batch.node.ubuntu 24.04" + + # The Platform requires a pool's task slots to equal the node's vCPU count. + # Azure VM size names embed that count (Standard_E8ds_v5 -> 8), so derive it + # from the size rather than hardcoding a number that can drift out of sync. + head_vm_cores = tonumber(regex("^Standard_[A-Z]+([0-9]+)", local.head_vm_size)[0]) + worker_vm_cores = tonumber(regex("^Standard_[A-Z]+([0-9]+)", local.worker_vm_size)[0]) + node_start_task_script = <<-SCRIPT + set -euo pipefail + + # azcopy: Nextflow's Azure Batch executor uses it to transfer files. + curl -sL https://aka.ms/downloadazcopy-v10-linux -o /tmp/azcopy.tgz + tar -xzf /tmp/azcopy.tgz --strip-components=1 -C /tmp + mkdir -p "$AZ_BATCH_NODE_SHARED_DIR/bin/" + cp /tmp/azcopy "$AZ_BATCH_NODE_SHARED_DIR/bin/" + + # AppArmor profile for Fusion containers on Ubuntu 24.04+. + mkdir -p /etc/apparmor.d/containers + printf '%s\n' \ + 'abi ,' \ + 'include ' \ + '' \ + 'profile seqera-fusionfs-container flags=(default_allow) {' \ + ' userns,' \ + ' mount fstype=fuse.fusion -> /fusion/,' \ + ' mount fstype=fuse.fusion -> /fusion/**,' \ + ' umount,' \ + ' include ' \ + ' include ' \ + ' include if exists ' \ + '}' > /etc/apparmor.d/containers/seqera-fusionfs-container + apparmor_parser -r /etc/apparmor.d/containers/seqera-fusionfs-container + SCRIPT + + # Batch parses the command line itself (no shell), so multi-line scripts with + # quoting don't survive. Base64-encode the script and decode it on the node. + pool_start_command = "/bin/bash -c 'echo ${base64encode(local.node_start_task_script)} | base64 -d | bash'" +} + +# Keeps pool names unique across re-creates. +resource "random_string" "suffix" { + length = 6 + special = false + upper = false +} + +# Head pool: runs the Nextflow head job. One node is enough. +resource "azurerm_batch_pool" "head" { + name = "rnaseq-head-${random_string.suffix.result}" + display_name = "rnaseq head pool" + resource_group_name = var.batch_account_rg + account_name = var.batch_account_name + vm_size = local.head_vm_size + node_agent_sku_id = local.node_agent_sku_id + max_tasks_per_node = local.head_vm_cores + + fixed_scale { + target_dedicated_nodes = 1 + } + + storage_image_reference { + publisher = "microsoft-dsvm" + offer = "ubuntu-hpc" + sku = "2404" + version = "latest" + } + + container_configuration { + type = "DockerCompatible" + } + + # Install azcopy and load the Fusion AppArmor profile on each node at startup. + start_task { + command_line = local.pool_start_command + task_retry_maximum = 1 + wait_for_success = true + + user_identity { + auto_user { + elevation_level = "Admin" + scope = "Pool" + } + } + } +} + +# Worker pool: runs the pipeline tasks. Autoscales 0..worker_max_nodes. +resource "azurerm_batch_pool" "worker" { + name = "rnaseq-worker-${random_string.suffix.result}" + display_name = "rnaseq worker pool" + resource_group_name = var.batch_account_rg + account_name = var.batch_account_name + vm_size = local.worker_vm_size + node_agent_sku_id = local.node_agent_sku_id + max_tasks_per_node = local.worker_vm_cores + + auto_scale { + evaluation_interval = "PT5M" + formula = <<-FORMULA + pending = avg($PendingTasks.GetSample(180 * TimeInterval_Second)); + $TargetDedicatedNodes = min(${local.worker_max_nodes}, pending); + $NodeDeallocationOption = taskcompletion; + FORMULA + } + + storage_image_reference { + publisher = "microsoft-dsvm" + offer = "ubuntu-hpc" + sku = "2404" + version = "latest" + } + + container_configuration { + type = "DockerCompatible" + } + + # Install azcopy and load the Fusion AppArmor profile on each node at startup. + start_task { + command_line = local.pool_start_command + task_retry_maximum = 1 + wait_for_success = true + + user_identity { + auto_user { + elevation_level = "Admin" + scope = "Pool" + } + } + } + + # Terraform manages the pool, not its live scale. + lifecycle { + ignore_changes = [auto_scale, fixed_scale] + } +} + +# Azure credentials already stored in the Platform. We look them up by name +# rather than creating them, so no keys appear in this config. +data "seqera_credentials" "all" { + workspace_id = var.workspace_id +} + +locals { + azure_credentials_id = one([ + for c in data.seqera_credentials.all.credentials : c.id + if c.name == var.azure_credential_name + ]) +} + +# Manual Azure Batch compute environment: uses the head pool, routes tasks to +# the worker pool. References to the pool names make Terraform create them first. +resource "seqera_compute_env" "main" { + workspace_id = var.workspace_id + + compute_env = { + name = "azure-batch-manual" + description = "Azure Batch CE on Terraform-managed pools" + platform = "azure-batch" + credentials_id = local.azure_credentials_id + + + config = { + azure_batch = { + region = var.region + work_dir = var.work_dir + head_pool = azurerm_batch_pool.head.name + worker_pool = azurerm_batch_pool.worker.name + enable_wave = true + enable_fusion = true + } + } + } +} + +# The Maintain tier (terraform/pipeline) needs this compute_env_id. +output "compute_env_id" { + description = "ID of the Azure Batch compute environment." + value = seqera_compute_env.main.compute_env_id +} diff --git a/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example b/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example new file mode 100644 index 0000000000..532f993e13 --- /dev/null +++ b/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example @@ -0,0 +1,20 @@ +# Copy to terraform.tfvars and fill in. terraform.tfvars is gitignored. +# Do NOT put your access token here. Export TOWER_ACCESS_TOKEN in the shell, +# and run `az login` first. + +subscription_id = "00000000-0000-0000-0000-000000000000" +workspace_id = 123456789 +region = "eastus" +work_dir = "az://work" + +# Existing Azure Batch account (referenced, not created). +batch_account_name = "mybatch" +batch_account_rg = "batch-rg" + +# Azure credentials already stored in the Platform workspace, referenced by name. +azure_credential_name = "my-azure-credentials" + +# Pool sizing lives in main.tf (locals), not here. + +# server_url defaults to Seqera Cloud; uncomment only for Enterprise. +# server_url = "https://platform.example.com/api" diff --git a/side-quests/platform_automation/terraform/compute-env/variables.tf b/side-quests/platform_automation/terraform/compute-env/variables.tf new file mode 100644 index 0000000000..c6ff497d28 --- /dev/null +++ b/side-quests/platform_automation/terraform/compute-env/variables.tf @@ -0,0 +1,47 @@ +# Inputs for the Admin tier compute-environment config. The Platform token is +# NOT here; it comes from TOWER_ACCESS_TOKEN. The Azure credentials are not here +# either: they are already stored in the Platform and referenced by name. + +variable "server_url" { + description = "Seqera Platform API endpoint. Cloud by default." + type = string + default = "https://api.cloud.seqera.io" +} + +variable "subscription_id" { + description = "Azure subscription ID." + type = string +} + +variable "workspace_id" { + description = "Numeric ID of the Platform workspace that owns the compute environment." + type = number +} + +# Existing Azure Batch account (referenced via a data source, not created). +variable "batch_account_name" { + description = "Name of the existing Azure Batch account." + type = string +} + +variable "batch_account_rg" { + description = "Resource group of the existing Azure Batch account." + type = string +} + +# Azure credentials already stored in the Platform (added to the workspace +# beforehand). Referenced by name; the keys never appear in this config. +variable "azure_credential_name" { + description = "Name of the existing Azure credentials in the Platform workspace." + type = string +} + +variable "region" { + description = "Azure region code of the Batch account, e.g. uaenorth." + type = string +} + +variable "work_dir" { + description = "Azure Blob Storage work directory, e.g. az://work." + type = string +} diff --git a/side-quests/platform_automation/terraform/pipeline/main.tf b/side-quests/platform_automation/terraform/pipeline/main.tf new file mode 100644 index 0000000000..196ceb1652 --- /dev/null +++ b/side-quests/platform_automation/terraform/pipeline/main.tf @@ -0,0 +1,44 @@ +# Maintain tier: add one rnaseq-nf pipeline to the Launchpad. This ADDS the +# pipeline; it does not launch it. Terraform manages existence, not runs: +# +# terraform apply creates rnaseq-nf- +# terraform apply (again) no-op; idempotent, never launches a run +# terraform apply -destroy removes it +# +# Launching is imperative, done with seqerakit/tw (see ../../seqerakit). +# +# Auth: export TOWER_ACCESS_TOKEN; the seqera provider reads it from the env. + +terraform { + required_version = ">= 1.6" + + required_providers { + seqera = { + source = "seqeralabs/seqera" + version = "0.40.1" + } + } +} + +provider "seqera" { + server_url = var.server_url +} + +resource "seqera_pipeline" "rnaseq_nf" { + name = "rnaseq-nf-${var.username}" + description = "rnaseq-nf for ${var.username}, added with Terraform" + workspace_id = var.workspace_id + + launch = { + pipeline = "https://github.com/nextflow-io/rnaseq-nf" + # The master branch moves; pin a release tag (e.g. v2.4) for reproducible launches. + revision = "master" + compute_env_id = var.compute_env_id + work_dir = var.work_dir + } +} + +output "pipeline_name" { + description = "Launchpad pipeline name. Use this when you launch it." + value = seqera_pipeline.rnaseq_nf.name +} diff --git a/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example b/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example new file mode 100644 index 0000000000..c8cca42dec --- /dev/null +++ b/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example @@ -0,0 +1,18 @@ +# Copy to terraform.tfvars and fill in. terraform.tfvars is gitignored. +# Do NOT put your access token here. Export TOWER_ACCESS_TOKEN in the shell. + +# Your workshop handle (letters, numbers, dash, underscore; no dots). +# Or pass it on the command line: terraform apply -var="username=$WORKSHOP_USER" +username = "your-handle" + +# Shared workshop workspace ID (the presenter gives you this). +workspace_id = 123456789 + +# Shared compute environment ID (the presenter gives you this). +compute_env_id = "REPLACE_ME" + +# Optional: defaults to az://work +# work_dir = "az://work" + +# server_url defaults to Seqera Cloud; uncomment only for Enterprise. +# server_url = "https://platform.example.com/api" diff --git a/side-quests/platform_automation/terraform/pipeline/variables.tf b/side-quests/platform_automation/terraform/pipeline/variables.tf new file mode 100644 index 0000000000..1bfdd9835c --- /dev/null +++ b/side-quests/platform_automation/terraform/pipeline/variables.tf @@ -0,0 +1,40 @@ +# Inputs for the Maintain tier pipeline config. + +variable "server_url" { + description = "Seqera Platform API endpoint. Cloud by default." + type = string + default = "https://api.cloud.seqera.io" +} + +# Your workshop handle makes the pipeline name unique in a shared workspace. No +# default on purpose, so you must set it. Set it with -var or in terraform.tfvars: +# +# terraform apply -var="username=$WORKSHOP_USER" +variable "username" { + description = "Your workshop handle. Makes the Launchpad pipeline name unique." + type = string + + # No dots: Launchpad pipeline names reject them. + validation { + condition = can(regex("^[a-zA-Z0-9_-]{2,80}$", var.username)) + error_message = "username must be 2-80 chars: letters, numbers, dash, underscore (no dots)." + } +} + +variable "workspace_id" { + description = "Numeric ID of the shared workshop workspace." + type = number +} + +# The Admin tier built the shared compute environment with terraform/compute-env +# and shared its ID. This is a string ID, not the numeric workspace ID. +variable "compute_env_id" { + description = "ID of the shared Azure Batch compute environment to launch against." + type = string +} + +variable "work_dir" { + description = "Azure Blob Storage work directory for runs of this pipeline." + type = string + default = "az://work" +}