|
3 | 3 | from __future__ import annotations |
4 | 4 |
|
5 | 5 | import hashlib |
| 6 | +import json |
6 | 7 | import random |
7 | 8 | from collections.abc import Sequence |
8 | 9 |
|
9 | | -from pydantic import BaseModel, ConfigDict, Field |
| 10 | +from pydantic import BaseModel, ConfigDict, Field, model_validator |
10 | 11 |
|
11 | | -from .models import EVAL_SCHEMA_VERSION, RunOutcome, RunRecord, SchemaVersion |
| 12 | +from .models import EVAL_SCHEMA_VERSION, RunOutcome, RunRecord, SchemaVersion, Sha256Digest |
12 | 13 |
|
13 | 14 |
|
14 | 15 | class BlindModel(BaseModel): |
@@ -43,6 +44,37 @@ class BlindMap(BlindModel): |
43 | 44 | """Separate restricted identity artifact, never included in grader output.""" |
44 | 45 |
|
45 | 46 | entries: tuple[BlindMapEntry, ...] |
| 47 | + blind_map_hash: Sha256Digest | None = None |
| 48 | + |
| 49 | + @model_validator(mode="after") |
| 50 | + def validate_hash(self) -> BlindMap: |
| 51 | + if self.blind_map_hash is not None and self.blind_map_hash != hash_blind_map(self.entries): |
| 52 | + raise ValueError("blind map hash does not match its entries") |
| 53 | + return self |
| 54 | + |
| 55 | + |
| 56 | +class GraderQueueOutput(BlindModel): |
| 57 | + """One normalized successful output safe for paid human grading.""" |
| 58 | + |
| 59 | + opaque_output_id: str = Field(pattern=r"^output_[0-9a-f]{24}$") |
| 60 | + presentation: str = Field(min_length=1) |
| 61 | + |
| 62 | + |
| 63 | +class GraderQueue(BlindModel): |
| 64 | + """Successful-only grader queue with no execution labels or outcomes.""" |
| 65 | + |
| 66 | + outputs: tuple[GraderQueueOutput, ...] |
| 67 | + |
| 68 | + |
| 69 | +def hash_blind_map(entries: Sequence[BlindMapEntry]) -> str: |
| 70 | + """Hash a restricted identity map in its frozen queue order.""" |
| 71 | + |
| 72 | + canonical = json.dumps( |
| 73 | + [entry.model_dump(mode="json") for entry in entries], |
| 74 | + sort_keys=True, |
| 75 | + separators=(",", ":"), |
| 76 | + ).encode("utf-8") |
| 77 | + return f"sha256:{hashlib.sha256(canonical).hexdigest()}" |
46 | 78 |
|
47 | 79 |
|
48 | 80 | def _opaque_id(*, planned_run_id: str, seed: int) -> str: |
@@ -76,3 +108,38 @@ def blind_run_records( |
76 | 108 | BlindedOutputSet(outputs=tuple(output for output, _ in blinded)), |
77 | 109 | BlindMap(entries=tuple(entry for _, entry in blinded)), |
78 | 110 | ) |
| 111 | + |
| 112 | + |
| 113 | +def build_grader_queue( |
| 114 | + records: Sequence[RunRecord], *, seed: int, forbidden_labels: Sequence[str] |
| 115 | +) -> tuple[GraderQueue, BlindMap]: |
| 116 | + """Normalize, leak-check, and blind successful outputs for paid grading.""" |
| 117 | + |
| 118 | + if len({record.planned_run_id for record in records}) != len(records): |
| 119 | + raise ValueError("run records must have unique planned_run_id values") |
| 120 | + blinded = [] |
| 121 | + labels = tuple(label.casefold() for label in forbidden_labels if label) |
| 122 | + for record in records: |
| 123 | + if record.outcome != "success": |
| 124 | + continue |
| 125 | + if record.output is None: |
| 126 | + raise ValueError("successful grader-queue records require output") |
| 127 | + presentation = " ".join(record.output.split()) |
| 128 | + if not presentation: |
| 129 | + raise ValueError("successful grader-queue records require nonempty output") |
| 130 | + folded = presentation.casefold() |
| 131 | + if any(label in folded for label in labels): |
| 132 | + raise ValueError("grader presentation contains a forbidden label") |
| 133 | + opaque_id = _opaque_id(planned_run_id=record.planned_run_id, seed=seed) |
| 134 | + blinded.append( |
| 135 | + ( |
| 136 | + GraderQueueOutput(opaque_output_id=opaque_id, presentation=presentation), |
| 137 | + BlindMapEntry(opaque_output_id=opaque_id, planned_run_id=record.planned_run_id), |
| 138 | + ) |
| 139 | + ) |
| 140 | + random.Random(seed).shuffle(blinded) |
| 141 | + entries = tuple(entry for _, entry in blinded) |
| 142 | + return ( |
| 143 | + GraderQueue(outputs=tuple(output for output, _ in blinded)), |
| 144 | + BlindMap(entries=entries, blind_map_hash=hash_blind_map(entries)), |
| 145 | + ) |
0 commit comments