From 4a8437aef17cdc32106a60858e90ba9d6afb16cc Mon Sep 17 00:00:00 2001 From: chriscarrollsmith Date: Thu, 11 Dec 2025 20:54:53 -0500 Subject: [PATCH 1/2] feat: add connection pooling support via USE_POOL toggle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add USE_POOL env var to switch between direct and pooled DB connections - Support pool-specific vars: DB_POOL_PORT, DB_POOL_NAME, DB_APPUSER, DB_APPUSER_PASSWORD - Add DB_SSLMODE support (defaults to "prefer") - Validate required env vars based on connection mode - Add tests for direct mode, pooled mode, and missing var validation - Document connection pooling setup in deployment docs Closes #143 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .env.example | 16 +++++++-- docs/deployment.qmd | 58 +++++++++++++++++++++++++++++- tests/utils/test_db.py | 81 +++++++++++++++++++++++++++++++++++++++++- utils/core/db.py | 55 +++++++++++++++++++++++----- 4 files changed, 198 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index b59dcee..41c1a58 100644 --- a/.env.example +++ b/.env.example @@ -4,13 +4,25 @@ SECRET_KEY= # Base URL BASE_URL=http://localhost:8000 -# Database +# Database connection mode (0 = direct, 1 = pooled) +USE_POOL=0 + +# Shared database settings +DB_HOST=127.0.0.1 +DB_SSLMODE=prefer + +# Direct connection settings (when USE_POOL=0) DB_USER=postgres DB_PASSWORD=postgres -DB_HOST=127.0.0.1 DB_PORT=5432 DB_NAME= +# Pooled connection settings (when USE_POOL=1) +# DB_APPUSER= +# DB_APPUSER_PASSWORD= +# DB_POOL_PORT=6543 +# DB_POOL_NAME= + # Resend RESEND_API_KEY= EMAIL_FROM= \ No newline at end of file diff --git a/docs/deployment.qmd b/docs/deployment.qmd index 10daf8b..adb2a7a 100644 --- a/docs/deployment.qmd +++ b/docs/deployment.qmd @@ -188,4 +188,60 @@ modal secret create your-app-name-secret \ #### Testing the Deployment -Access the provided Modal URL in your browser. Browse the site and test the registration and password reset features to ensure database and Resend connections work. \ No newline at end of file +Access the provided Modal URL in your browser. Browse the site and test the registration and password reset features to ensure database and Resend connections work. + +## Connection Pooling + +When deploying to production with many concurrent connections or in serverless environments, you may want to use an external connection pooler like [PgBouncer](https://www.pgbouncer.org/), [Supabase Pooler](https://supabase.com/docs/guides/database/connecting-to-postgres#connection-pooler), or [AWS RDS Proxy](https://aws.amazon.com/rds/proxy/). + +Connection poolers sit between your application and database, managing a pool of persistent connections. This reduces connection overhead and allows your application to handle more concurrent requests without exhausting database connections. + +### Configuring Pooled Connections + +This template supports switching between direct and pooled database connections via the `USE_POOL` environment variable. + +**Direct mode (default):** Set `USE_POOL=0` or leave it unset. The application connects directly to the database using: + +- `DB_HOST` - Database host +- `DB_PORT` - Database port (typically 5432) +- `DB_NAME` - Database name +- `DB_USER` - Database username +- `DB_PASSWORD` - Database password + +**Pooled mode:** Set `USE_POOL=1`. The application connects via the pooler using: + +- `DB_HOST` - Pooler host (may be the same as the database host) +- `DB_POOL_PORT` - Pooler port (typically 6543 for PgBouncer) +- `DB_POOL_NAME` - Database name for pooled connections +- `DB_APPUSER` - Application-specific database user +- `DB_APPUSER_PASSWORD` - Application user password + +Both modes support `DB_SSLMODE` (defaults to `prefer`) for configuring SSL connections. + +### Example: Supabase with Connection Pooling + +Supabase provides a built-in PgBouncer pooler. To use it: + +1. In your Supabase dashboard, go to Settings > Database +2. Find the "Connection pooling" section and copy the pooler connection string +3. Extract the host, port, database, user, and password from the connection string +4. Configure your environment: + +```bash +USE_POOL=1 +DB_HOST=aws-0-us-east-1.pooler.supabase.com +DB_POOL_PORT=6543 +DB_POOL_NAME=postgres +DB_APPUSER=postgres.yourproject +DB_APPUSER_PASSWORD=your-password +DB_SSLMODE=require +``` + +### When to Use Connection Pooling + +Consider using a connection pooler when: + +- Running in serverless environments (Modal, AWS Lambda, Vercel) where cold starts create many new connections +- Your application handles many concurrent requests +- You're hitting database connection limits +- You want to reduce connection latency for frequently-accessed queries \ No newline at end of file diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py index 7b86377..d6f2b0f 100644 --- a/tests/utils/test_db.py +++ b/tests/utils/test_db.py @@ -1,4 +1,4 @@ - +import pytest from sqlmodel import Session, select, inspect from sqlalchemy import Engine from utils.core.db import ( @@ -12,6 +12,10 @@ from utils.core.models import Role, Permission, Organization, RolePermissionLink, ValidPermissions from tests.conftest import SetupError + +# --- Connection URL Tests --- + + def test_get_connection_url(env_vars): """Test that get_connection_url returns a valid URL object""" url = get_connection_url() @@ -19,6 +23,81 @@ def test_get_connection_url(env_vars): assert url.database is not None +def test_get_connection_url_direct_mode(monkeypatch): + """Test that direct mode uses standard DB vars.""" + # Clear any existing vars + for var in ["USE_POOL", "DB_HOST", "DB_PORT", "DB_NAME", "DB_USER", "DB_PASSWORD", + "DB_POOL_PORT", "DB_POOL_NAME", "DB_APPUSER", "DB_APPUSER_PASSWORD"]: + monkeypatch.delenv(var, raising=False) + + # Set direct mode vars + monkeypatch.setenv("DB_HOST", "localhost") + monkeypatch.setenv("DB_PORT", "5432") + monkeypatch.setenv("DB_NAME", "testdb") + monkeypatch.setenv("DB_USER", "testuser") + monkeypatch.setenv("DB_PASSWORD", "testpass") + + url = get_connection_url() + assert url.host == "localhost" + assert url.port == 5432 + assert url.database == "testdb" + assert url.username == "testuser" + assert url.query.get("sslmode") == "prefer" + + +def test_get_connection_url_pooled_mode(monkeypatch): + """Test that pooled mode uses pool-specific vars.""" + # Clear any existing vars + for var in ["USE_POOL", "DB_HOST", "DB_PORT", "DB_NAME", "DB_USER", "DB_PASSWORD", + "DB_POOL_PORT", "DB_POOL_NAME", "DB_APPUSER", "DB_APPUSER_PASSWORD"]: + monkeypatch.delenv(var, raising=False) + + # Set pooled mode vars + monkeypatch.setenv("USE_POOL", "1") + monkeypatch.setenv("DB_HOST", "pooler.example.com") + monkeypatch.setenv("DB_POOL_PORT", "6543") + monkeypatch.setenv("DB_POOL_NAME", "pooldb") + monkeypatch.setenv("DB_APPUSER", "appuser") + monkeypatch.setenv("DB_APPUSER_PASSWORD", "apppass") + monkeypatch.setenv("DB_SSLMODE", "require") + + url = get_connection_url() + assert url.host == "pooler.example.com" + assert url.port == 6543 + assert url.database == "pooldb" + assert url.username == "appuser" + assert url.query.get("sslmode") == "require" + + +def test_get_connection_url_missing_direct_vars(monkeypatch): + """Test that missing direct mode vars raises ValueError.""" + # Clear all DB vars + for var in ["USE_POOL", "DB_HOST", "DB_PORT", "DB_NAME", "DB_USER", "DB_PASSWORD", + "DB_POOL_PORT", "DB_POOL_NAME", "DB_APPUSER", "DB_APPUSER_PASSWORD"]: + monkeypatch.delenv(var, raising=False) + + with pytest.raises(ValueError, match="Missing environment variables"): + get_connection_url() + + +def test_get_connection_url_missing_pool_vars(monkeypatch): + """Test that missing pooled mode vars raises ValueError.""" + # Clear all DB vars + for var in ["USE_POOL", "DB_HOST", "DB_PORT", "DB_NAME", "DB_USER", "DB_PASSWORD", + "DB_POOL_PORT", "DB_POOL_NAME", "DB_APPUSER", "DB_APPUSER_PASSWORD"]: + monkeypatch.delenv(var, raising=False) + + monkeypatch.setenv("USE_POOL", "1") + monkeypatch.setenv("DB_HOST", "localhost") + # Missing: DB_POOL_PORT, DB_POOL_NAME, DB_APPUSER, DB_APPUSER_PASSWORD + + with pytest.raises(ValueError, match="Missing environment variables.*DB_POOL_PORT"): + get_connection_url() + + +# --- Permission and Role Tests --- + + def test_create_permissions(session: Session): """Test that create_permissions creates all ValidPermissions""" # Clear existing permissions diff --git a/utils/core/db.py b/utils/core/db.py index 487fbea..3e2c514 100644 --- a/utils/core/db.py +++ b/utils/core/db.py @@ -36,23 +36,62 @@ def get_connection_url() -> URL: """ Constructs a SQLModel URL object for connecting to the PostgreSQL database. - The connection details are sourced from environment variables, which should include: + Supports two connection modes controlled by the USE_POOL environment variable: + - Direct mode (USE_POOL=0, default): Connects directly to the database + - Pooled mode (USE_POOL=1): Connects via an external connection pooler (e.g., PgBouncer) + + Direct mode environment variables: + - DB_HOST: Database host address + - DB_PORT: Database port + - DB_NAME: Database name - DB_USER: Database username - DB_PASSWORD: Database password + - DB_SSLMODE: SSL mode (default: "prefer") + + Pooled mode environment variables: - DB_HOST: Database host address - - DB_PORT: Database port (default is 5432) - - DB_NAME: Database name + - DB_POOL_PORT: Connection pooler port + - DB_POOL_NAME: Database name for pooled connections + - DB_APPUSER: Application user for pooled connections + - DB_APPUSER_PASSWORD: Application user password + - DB_SSLMODE: SSL mode (default: "prefer") Returns: URL: A SQLModel URL object containing the connection details. + + Raises: + ValueError: If required environment variables are missing. """ + use_pool = bool(int(os.getenv("USE_POOL", "0"))) + + host = os.getenv("DB_HOST") + sslmode = os.getenv("DB_SSLMODE", "prefer") + + if use_pool: + port = os.getenv("DB_POOL_PORT") + database = os.getenv("DB_POOL_NAME") + username = os.getenv("DB_APPUSER") + password = os.getenv("DB_APPUSER_PASSWORD") + required = ["DB_HOST", "DB_POOL_PORT", "DB_POOL_NAME", "DB_APPUSER", "DB_APPUSER_PASSWORD"] + else: + port = os.getenv("DB_PORT") + database = os.getenv("DB_NAME") + username = os.getenv("DB_USER") + password = os.getenv("DB_PASSWORD") + required = ["DB_HOST", "DB_PORT", "DB_NAME", "DB_USER", "DB_PASSWORD"] + + missing = [var for var in required if not os.getenv(var)] + if missing: + raise ValueError(f"Missing environment variables: {', '.join(missing)}") + database_url: URL = URL.create( drivername="postgresql", - username=os.getenv("DB_USER"), - password=os.getenv("DB_PASSWORD"), - host=os.getenv("DB_HOST"), - port=int(os.getenv("DB_PORT") or "5432"), - database=os.getenv("DB_NAME"), + username=username, + password=password, + host=host, + port=int(port), # type: ignore[arg-type] + database=database, + query={"sslmode": sslmode}, ) return database_url From 97ceddcf804fcee0020fea8ec3d4f946ca6a57a5 Mon Sep 17 00:00:00 2001 From: chriscarrollsmith Date: Thu, 11 Dec 2025 21:00:07 -0500 Subject: [PATCH 2/2] docs: add instructions for setting up restricted app user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents how to create a database user with limited permissions for use with pooled connections, following least-privilege principle. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- docs/deployment.qmd | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/deployment.qmd b/docs/deployment.qmd index adb2a7a..bf74953 100644 --- a/docs/deployment.qmd +++ b/docs/deployment.qmd @@ -237,6 +237,42 @@ DB_APPUSER_PASSWORD=your-password DB_SSLMODE=require ``` +### Setting Up a Restricted Application User + +When using connection pooling, it's a security best practice to create a separate database user with restricted permissions for your application's runtime connections. This follows the principle of least privilege - the app user can only perform the operations it needs, limiting potential damage from SQL injection or other vulnerabilities. + +Connect to your database as the admin user and run: + +```sql +-- Create the application user +CREATE USER appuser WITH PASSWORD 'your-secure-password'; + +-- Grant connect permission +GRANT CONNECT ON DATABASE yourdb TO appuser; + +-- Grant usage on the public schema +GRANT USAGE ON SCHEMA public TO appuser; + +-- Grant permissions on all existing tables +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO appuser; + +-- Grant permissions on sequences (needed for auto-incrementing IDs) +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO appuser; + +-- Ensure future tables also get these permissions +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO appuser; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO appuser; +``` + +Then configure your environment to use this user for pooled connections: + +```bash +DB_APPUSER=appuser +DB_APPUSER_PASSWORD=your-secure-password +``` + +Keep your admin credentials (`DB_USER`, `DB_PASSWORD`) for direct connections used during schema migrations or administrative tasks. + ### When to Use Connection Pooling Consider using a connection pooler when: