Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
ef8c44a
feat: add count-based work admission ceilings
vladmesh Aug 25, 2026
d50739a
test: cover count admission in scheduler fixtures
vladmesh Aug 25, 2026
3d915f8
fix: use unique work admission migration revision
vladmesh Aug 25, 2026
6f1dfcb
fix: seed work admission configs in test API containers
vladmesh Aug 25, 2026
2763e97
fix: isolate service tests from admission ceilings
vladmesh Aug 25, 2026
283cf72
fix: serialize configuration seeds in test stacks
vladmesh Aug 25, 2026
1dc3b43
test: prove concurrent project admission ceiling
vladmesh Aug 25, 2026
0b8b108
refactor: start paid runs atomically
vladmesh Aug 25, 2026
8aa0ce8
fix: preserve budget denial context in paid starts
vladmesh Aug 25, 2026
05e233e
test: route scheduler work through paid start command
vladmesh Aug 25, 2026
3bd85e2
fix: retain internal run fixture compatibility
vladmesh Aug 25, 2026
7a8dc61
fix: enforce paid run admission boundary
vladmesh Aug 25, 2026
8999bf4
test: start paid reconciliation runs canonically
vladmesh Aug 25, 2026
dc6ea39
test: prove paid run boundary behavior
vladmesh Aug 25, 2026
65f0044
test: create paid run fixtures canonically
vladmesh Aug 26, 2026
3071f83
test: start integration engineering runs canonically
vladmesh Aug 26, 2026
654bd11
fix: harden paid run admission command
vladmesh Aug 26, 2026
a0bb45c
fix: seed protected admission controls through typed API
vladmesh Aug 26, 2026
00adbe1
fix: initialize protected admission controls through typed API
vladmesh Aug 26, 2026
30c0f7b
test: isolate paid run admission race capacity
vladmesh Aug 26, 2026
ca267ee
fix: persist paid work refusal identity
vladmesh Aug 26, 2026
0656a0c
fix: serialize paid run command replays
vladmesh Aug 26, 2026
2dec2fc
test: assert normalized paid command audit payload
vladmesh Aug 26, 2026
dbb3664
fix: re-evaluate paid run retries atomically
vladmesh Aug 26, 2026
ac3bab5
fix: deliver canonical deploy-fix refusal text
vladmesh Aug 26, 2026
ce2952d
fix: durably deliver scheduler admission refusals
vladmesh Aug 26, 2026
88555e5
fix: close unpublished paid runs atomically
vladmesh Aug 26, 2026
c6a230f
fix: park tasks after paid admission refusal
vladmesh Aug 26, 2026
ea5d8bf
test: prove released deploy fix re-enters admission
vladmesh Aug 26, 2026
f2ffa69
test: accept created deploy fix budget policy
vladmesh Aug 26, 2026
b3f0791
test: use valid deploy fix retry references
vladmesh Aug 26, 2026
1b0184b
fix: reject terminal paid run identity reuse
vladmesh Aug 26, 2026
480cb67
test: make admission parking stateful
vladmesh Aug 26, 2026
cab7eee
fix: atomically abort scheduler handoff failures
vladmesh Aug 26, 2026
2cf845b
test: assert atomic scheduler handoff abort
vladmesh Aug 26, 2026
02b60a8
test: model terminal runs after atomic abort
vladmesh Aug 26, 2026
0812dfd
test: model failed aborted engineering run
vladmesh Aug 26, 2026
c2220c3
fix: ignore aborted pre-handoff runs on retry
vladmesh Aug 26, 2026
c26b98b
fix: preserve terminal identity error precedence
vladmesh Aug 26, 2026
700070c
test: cover paid run terminal and operator boundaries
vladmesh Aug 26, 2026
5b3f282
fix: preserve uncertain paid handoffs
vladmesh Aug 26, 2026
17849ae
test: prove aborted dispatch recovery end to end
vladmesh Aug 26, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docker/test/integration/backend-dind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,8 @@ services:
SOURCE_HASH: ${WORKER_SOURCE_HASH:-}
# Uses default entrypoint.sh which runs alembic migrations before uvicorn
environment:
- ENVIRONMENT=test
- SEED_SYSTEM_CONFIGS_ON_START=true
- DATABASE_URL=postgresql+asyncpg://postgres:postgres@db:5432/postgres
- REDIS_URL=redis://redis:6379/0
- SECRETS_ENCRYPTION_KEY=wHhIQWmPfLt60oHdxzbQhY1ZKnUon12e5_SuZ33xDxc=
Expand Down
2 changes: 2 additions & 0 deletions docker/test/integration/backend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ services:
args:
SOURCE_HASH: ${WORKER_SOURCE_HASH:-}
environment:
- ENVIRONMENT=test
- SEED_SYSTEM_CONFIGS_ON_START=true
- DATABASE_URL=postgresql+asyncpg://postgres:postgres@db:5432/postgres
- REDIS_URL=redis://redis:6379/0
- SECRETS_ENCRYPTION_KEY=wHhIQWmPfLt60oHdxzbQhY1ZKnUon12e5_SuZ33xDxc=
Expand Down
2 changes: 2 additions & 0 deletions docker/test/integration/frontend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ services:
args:
SOURCE_HASH: ${WORKER_SOURCE_HASH:-}
environment:
- ENVIRONMENT=test
- SEED_SYSTEM_CONFIGS_ON_START=true
- DATABASE_URL=postgresql+asyncpg://postgres:postgres@db:5432/postgres
- REDIS_URL=redis://redis:6379/0
- LK_JWT_SECRET=test-lk-jwt-secret
Expand Down
2 changes: 1 addition & 1 deletion docker/test/integration/infra.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ services:
SOURCE_HASH: ${WORKER_SOURCE_HASH:-}
command: ["/entrypoint.sh"]
environment:
- ENVIRONMENT=test
- DATABASE_URL=postgresql+asyncpg://postgres:postgres@db:5432/postgres
- REDIS_URL=redis://redis:6379/0
- ENVIRONMENT=test
- LK_JWT_SECRET=test-lk-jwt-secret
- INTERNAL_API_KEY=test-internal-api-key-for-integration-tests
depends_on:
Expand Down
2 changes: 2 additions & 0 deletions docker/test/integration/po-tools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ services:
SOURCE_HASH: ${WORKER_SOURCE_HASH:-}
command: ["/entrypoint.sh"]
environment:
- ENVIRONMENT=test
- SEED_SYSTEM_CONFIGS_ON_START=true
- DATABASE_URL=postgresql+asyncpg://postgres:postgres@db:5432/postgres
- REDIS_URL=redis://redis:6379/0
- ENVIRONMENT=test
Expand Down
2 changes: 2 additions & 0 deletions docker/test/service/api.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ services:
- DATABASE_URL=postgresql+asyncpg://postgres:postgres@db:5432/postgres
- REDIS_URL=redis://redis:6379/0
- ENVIRONMENT=test
- SEED_SYSTEM_CONFIGS_ON_START=true
- SYSTEM_CONFIGS_TEST_OVERLAY=/app/scripts/system_configs.service_test.yaml
- SECRETS_ENCRYPTION_KEY=wHhIQWmPfLt60oHdxzbQhY1ZKnUon12e5_SuZ33xDxc=
- LK_JWT_SECRET=test-lk-jwt-secret-for-service-tests
- INTERNAL_API_KEY=test-internal-api-key-for-service-tests
Expand Down
2 changes: 2 additions & 0 deletions docker/test/service/scheduler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ services:
SOURCE_HASH: ${WORKER_SOURCE_HASH:-}
command: ["/entrypoint.sh"]
environment:
- ENVIRONMENT=test
- SEED_SYSTEM_CONFIGS_ON_START=true
- DATABASE_URL=postgresql+asyncpg://postgres:postgres@db:5432/postgres
- REDIS_URL=redis://redis:6379/0
- ENVIRONMENT=test
Expand Down
20 changes: 20 additions & 0 deletions docs/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,26 @@
### Added

- One-time promo-code registration that atomically arms an enabled engineering budget policy.
- Count-based work admission for projects and concurrent engineering/QA runs,
with an internal/admin emergency-stop API and durable typed decision audit
records. Paid runs now start through one transactional API command, which
holds the counted slot through queued-Run creation and checks engineering
money only after the count gate.

### Fixed

- Updated paid-run test fixtures to use the transactional paid-run command;
fixtures that only require a Run record now use a non-paid type.
- Paid-run starts now replay an identical stable command idempotently and reject
conflicting payloads; the obsolete standalone paid-work admission oracle was
removed. Emergency-stop writes are strict booleans and admission controls are
protected from the generic configuration mutation API.
- Paid-work refusals now persist their command identity, project owner, typed
reason and Russian owner-facing text without caching a transient outcome.
- A paid-run retry now rechecks controls rather than caching a prior refusal;
it only reuses a live Run whose engineering reservation remains active.
- Scheduler admission refusals now park their Task as well as the Story, and a
handled pre-handoff publish failure closes the Run with its released hold.

## 2026-08-24

Expand Down
68 changes: 61 additions & 7 deletions docs/CONTRACTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,59 @@ are hard-deleted; all accounting facts and the resolved `user_id` remain immutab

### Engineering budget policies

### Count-based work admission

`work_admission` is independent of engineering money. Project creation uses its
per-user lock directly. Paid coding-agent work uses `POST /api/work-admission/paid-runs`:
the command locks the controls, checks the emergency stop and concurrent-run
ceiling, performs engineering's existing money admission when applicable, then
creates the queued Run before the transaction commits. No separate successful
paid-work admission exists. Every count decision is stored in
`work_admission_audits` with a typed outcome and reason.

Paid-run retries are decided only by that command while it holds the admission
control locks. An existing Run is replayed only when it is `queued` or
`running` and, for a budgeted engineering attempt, its reservation is still
`active`. Audit rows preserve command identity for payload-conflict detection;
they never cache a denial. A retry after a stop is lifted or a capacity slot is
freed therefore evaluates the controls again. A terminal Run is never reopened:
its identity returns the typed `paid_run_identity_expired` conflict and a caller
must create a new attempt identity. A changed payload under a terminal identity
returns `paid_run_command_conflict` before that expiry outcome.

For scheduler dispatch and QA handoff, an addressable refusal writes its owed
owner-notification record before the task/story transitions that park work;
operator-facing spawn-worker and run-e2e instead return the typed result
synchronously, while the command still records its audit reason.

Known limitations: if a process dies after the paid-run command commits and
before handoff, its queued Run continues to occupy the ceiling until manual
intervention. The atomic internal abort command is used only when preparation
failed before any queue call; it marks the Run `cancelled` and releases its
hold. A publication exception has an unknown broker outcome, so the queued Run
and its active hold remain for normal unfinished-run recovery rather than being
incorrectly cancelled.
Refusal notification is attached to the project's initiating Run, so a second
task refusal for that project can be suppressed and a standalone task has no
owner notification.

The deployed defaults in `scripts/system_configs.yaml` are:

- `work_admission.max_projects_per_user=3`, measured as non-archived projects
owned by one non-admin user. Deleted projects are absent and archived projects
do not count; administrators are unlimited.
- `work_admission.max_concurrent_paid_runs=5`, measured globally as queued or
running `engineering` and `qa` runs together.
- `work_admission.emergency_stop=false`. Internal/admin callers read and set
this one operator switch at `/api/work-admission/emergency-stop`; while true,
no new project, engineering run, or QA run is admitted. The stored value must
be a boolean; malformed configuration fails closed.
It never changes existing rows, workspaces, containers, or runs.

Engineering's count check precedes its existing monetary
`admit_engineering_attempt` inside that same command, so a count-based refusal
cannot create a financial reservation. QA never enters the monetary gate.

`engineering_budget_policies` holds at most one durable policy row per `user_id`.
`limit_microusd` is a non-negative integer number of micro-USD; budget requests never
accept floating-point or dollar-denominated money. `state` is the typed
Expand Down Expand Up @@ -120,13 +173,14 @@ denies. `POST .../admissions/{attempt_id}/release` may release only a proven pre
`active` hold.

`engineering_budget_reservations` records those decisions separately from the immutable
ledger. The pre-handoff boundary ends only when the engineering message has published.
Dispatchers validate cheap local conditions first; after an admitted `active` hold, every
exception or typed refusal before that boundary, including Run creation, recipient
resolution and publishing, changes it to `released`. A released row proves only that its
previous handoff did not begin; a deterministic replay such as a deploy-fix dispatch must
re-enter admission and obtain a newly `active` row before any story transition, Run creation
or queue publication. This applies to ordinary task
ledger. The pre-handoff boundary ends before an engineering message is submitted to the
queue. Dispatchers validate cheap local conditions first; after an admitted `active` hold,
a typed refusal or an exception proven to occur before that queue call — including Run
creation and recipient resolution — changes it to `released`. An exception from publication
has an unknown broker outcome and does not release the hold or cancel the queued Run. A
released row proves only that its previous handoff did not begin; a deterministic replay such
as a deploy-fix dispatch must re-enter admission and obtain a newly `active` row before any
story transition, Run creation or queue publication. This applies to ordinary task
dispatch and supervisor deploy-fix dispatch, whose stable attempt id is
`eng-deploy-fix-{deploy_run_id}-{attempt}`. A scheduler denial has no Run or queue side
effect and moves the affected task or deploy-fix story to `waiting_human_review` with
Expand Down
16 changes: 14 additions & 2 deletions scripts/seed_system_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@
from shared.clients.internal_api import InternalAPISyncClient

CONFIG_PATH = Path(__file__).resolve().parent / "system_configs.yaml"
_WORK_ADMISSION_KEYS = {
"work_admission.emergency_stop",
"work_admission.max_projects_per_user",
"work_admission.max_concurrent_paid_runs",
}


def load_configs(path: Path) -> list[dict]:
Expand Down Expand Up @@ -92,8 +97,15 @@ def seed_system_configs(api_base_url: str, configs_path: Path) -> bool:
"description": config["description"],
"updated_by": "seed",
}
resp = client.request_raw("POST", "system-configs/", json=payload)
if resp.status_code != httpx.codes.CREATED:
if key in _WORK_ADMISSION_KEYS:
resp = client.request_raw(
"PUT", f"work-admission/controls/{key}", json={"value": value}
)
expected_status = httpx.codes.OK
else:
resp = client.request_raw("POST", "system-configs/", json=payload)
expected_status = httpx.codes.CREATED
if resp.status_code != expected_status:
print(f" Failed to write '{key}': {resp.status_code} - {resp.text}")
success = False
continue
Expand Down
20 changes: 20 additions & 0 deletions scripts/system_configs.service_test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Overrides for the shared API service-test database.
#
# Service tests intentionally retain records across the suite to exercise APIs
# against an evolving dataset. These ceilings remain enabled in the test
# contour, but use explicit high values so unrelated scenarios do not exhaust
# an operational production limit. Admission tests set their own low value.
- key: work_admission.max_projects_per_user
value: 10000
category: work_admission
description: "Service-test maximum non-archived projects for one non-admin user"

- key: work_admission.max_concurrent_paid_runs
value: 10000
category: work_admission
description: "Service-test maximum queued or running engineering and QA coding-agent runs"

- key: work_admission.emergency_stop
value: false
category: work_admission
description: "Service-test operator switch that blocks new admitted work"
15 changes: 15 additions & 0 deletions scripts/system_configs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,21 @@
# the database alone and are never touched.

# --- scheduler ---
- key: work_admission.max_projects_per_user
value: 3
category: work_admission
description: "Maximum non-archived projects for one non-admin user"

- key: work_admission.max_concurrent_paid_runs
value: 5
category: work_admission
description: "Maximum queued or running engineering and QA coding-agent runs"

- key: work_admission.emergency_stop
value: false
category: work_admission
description: "Operator switch that blocks new projects, engineering and QA work"

- key: scheduler.dispatch_interval_seconds
value: 30
category: scheduler
Expand Down
1 change: 1 addition & 0 deletions services/api/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ COPY services/api/src ./src
COPY services/api/migrations ./migrations
COPY services/api/alembic.ini ./alembic.ini
COPY services/api/entrypoint.sh /entrypoint.sh
COPY scripts/seed_system_configs.py scripts/system_configs.yaml scripts/system_configs.service_test.yaml /app/scripts/
RUN chmod +x /entrypoint.sh

# Expose port
Expand Down
18 changes: 18 additions & 0 deletions services/api/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,22 @@ set -e
cd /app
alembic upgrade head

if [ "$ENVIRONMENT" = "test" ] && [ "$SEED_SYSTEM_CONFIGS_ON_START" = "true" ]; then
uvicorn src.main:app --host 0.0.0.0 --port 8000 &
api_pid=$!
until curl --fail --silent http://localhost:8000/health >/dev/null; do
sleep 1
done
python /app/scripts/seed_system_configs.py \
--api-base-url http://localhost:8000 \
--configs-path /app/scripts/system_configs.yaml
if [ -n "$SYSTEM_CONFIGS_TEST_OVERLAY" ]; then
python /app/scripts/seed_system_configs.py \
--api-base-url http://localhost:8000 \
--configs-path "$SYSTEM_CONFIGS_TEST_OVERLAY"
fi
wait "$api_pid"
exit $?
fi

exec uvicorn src.main:app --host 0.0.0.0 --port 8000
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
"""Persist paid-work command identity and the owner-facing refusal text.

Revision ID: a3d4e5f6a7b8
Revises: f7e8d9c0b1a2
"""

from collections.abc import Sequence

from alembic import op
import sqlalchemy as sa

revision: str = "a3d4e5f6a7b8"
down_revision: str | None = "f7e8d9c0b1a2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None


def upgrade() -> None:
op.add_column("work_admission_audits", sa.Column("command_payload", sa.JSON(), nullable=True))
op.add_column("work_admission_audits", sa.Column("message", sa.Text(), nullable=True))


def downgrade() -> None:
op.drop_column("work_admission_audits", "message")
op.drop_column("work_admission_audits", "command_payload")
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
"""Add count-based work admission audit records.

Revision ID: f7e8d9c0b1a2
Revises: f6e7d8c9b0a1
"""

from collections.abc import Sequence

from alembic import op
import sqlalchemy as sa

revision: str = "f7e8d9c0b1a2"
down_revision: str | None = "f6e7d8c9b0a1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None


def upgrade() -> None:
op.create_table(
"work_admission_audits",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("subject", sa.String(length=32), nullable=False),
sa.Column("outcome", sa.String(length=16), nullable=False),
sa.Column("reason", sa.String(length=32), nullable=True),
sa.Column("user_id", sa.Integer(), nullable=True),
sa.Column("reference_id", sa.String(length=255), nullable=True),
sa.Column(
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
),
sa.Column(
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_work_admission_audits_subject", "work_admission_audits", ["subject"])
op.create_index("ix_work_admission_audits_outcome", "work_admission_audits", ["outcome"])
op.create_index("ix_work_admission_audits_user_id", "work_admission_audits", ["user_id"])
op.create_index(
"ix_work_admission_audits_reference_id", "work_admission_audits", ["reference_id"]
)


def downgrade() -> None:
op.drop_table("work_admission_audits")
1 change: 1 addition & 0 deletions services/api/src/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ async def root():
app.include_router(routers.runs.router, prefix="/api")
app.include_router(routers.engineering_budget_policies.router, prefix="/api")
app.include_router(routers.engineering_budget_policies.self_router, prefix="/api")
app.include_router(routers.work_admission.router, prefix="/api")
app.include_router(routers.tasks.router, prefix="/api")
app.include_router(routers.brainstorms.router, prefix="/api")
app.include_router(routers.repositories.router, prefix="/api")
Expand Down
2 changes: 2 additions & 0 deletions services/api/src/routers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
tasks,
temporary_access,
users,
work_admission,
)

__all__ = [
Expand Down Expand Up @@ -52,4 +53,5 @@
"tasks",
"temporary_access",
"users",
"work_admission",
]
Loading
Loading