Skip to content

Commit 4535315

Browse files
Build fixed-input candidate generation plans in Apps (#465)
* Build fixed-input candidate generation plans in Apps Signed-off-by: zengchuang <zengchuang3@huawei.com> Agent-assisted-by: Codex (author) * Match the Harness candidate planning port Signed-off-by: zengchuang <zengchuang3@huawei.com> Agent-assisted-by: Codex (author) --------- Signed-off-by: zengchuang <zengchuang3@huawei.com>
1 parent 64f8203 commit 4535315

3 files changed

Lines changed: 320 additions & 0 deletions

File tree

‎src/inferrsi/apps/api.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from .artifact_reader import ArtifactReader, AssessmentArtifacts
2424
from .bridge import bind_execution as _bind_prepared_execution
2525
from .bridge import check_assessment_identity, execution_fields, to_evidence, verify_protocol
26+
from .candidate_provider import CandidateProvider as CandidateProvider
2627
from .result import (
2728
assessment_result,
2829
execution_result,
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
"""Construct the Apps side of a generation plan for Harness to seal."""
2+
3+
from collections.abc import Callable
4+
from copy import deepcopy
5+
6+
from inferrsi.contracts.json_v1 import encode
7+
from inferrsi.harness.errors import PublicError
8+
9+
from .archive import _logical_path
10+
from .bridge import (
11+
_ATTEMPT_FIELDS,
12+
_definition,
13+
_id,
14+
_input_paths,
15+
_output_paths,
16+
_require_disjoint_paths,
17+
)
18+
from .result import validate_artifact_ref
19+
20+
_CONTEXT_FIELDS = {
21+
"campaign_id", "generation_id", "candidate", "parent", "task_materials",
22+
"delivered_context", "allowed_feedback", "output_contract",
23+
}
24+
_OUTPUT_CONTRACT_FIELDS = {
25+
"manifest_role", "candidate_role", "format_schema", "required_input_roles", "parser_revision",
26+
}
27+
28+
29+
class CandidateProvider:
30+
"""Apps adapter for planning the generation leg of a Harness loop."""
31+
32+
def __init__(
33+
self, *, provider: dict, output_contract: dict, steps: list[dict],
34+
bind_inputs: Callable[[dict], list[dict]], output_root_for: Callable[[str], str],
35+
required_capabilities: list[str],
36+
):
37+
self._provider = deepcopy(provider)
38+
self._output_contract = deepcopy(output_contract)
39+
self._steps = deepcopy(steps)
40+
self._bind_inputs = bind_inputs
41+
self._output_root_for = output_root_for
42+
self._required_capabilities = deepcopy(required_capabilities)
43+
44+
def plan_generation(self, context: dict, attempt: dict) -> dict:
45+
"""Bind the authorized context and original attempt to the configured worker."""
46+
return _plan_generation(
47+
context, attempt, provider=self._provider, output_contract=self._output_contract,
48+
steps=self._steps, bind_inputs=self._bind_inputs, output_root_for=self._output_root_for,
49+
required_capabilities=self._required_capabilities,
50+
)
51+
52+
53+
def _plan_generation(
54+
context: dict, attempt: dict, *, provider: dict, output_contract: dict,
55+
steps: list[dict], bind_inputs: Callable[[dict], list[dict]],
56+
output_root_for: Callable[[str], str], required_capabilities: list[str],
57+
) -> dict:
58+
"""Build the fixed-input plan for Harness to seal and execute."""
59+
try:
60+
if (
61+
not isinstance(context, dict) or context.keys() != _CONTEXT_FIELDS
62+
or not _id(context["campaign_id"]) or not _id(context["generation_id"])
63+
or not isinstance(attempt, dict) or attempt.keys() != _ATTEMPT_FIELDS
64+
or not _id(attempt["attempt_id"]) or attempt["request_id"] != attempt["attempt_id"]
65+
or not _definition(provider)
66+
):
67+
raise ValueError("expected a candidate context, provider and allocated attempt")
68+
if (
69+
not isinstance(output_contract, dict)
70+
or output_contract.keys() != _OUTPUT_CONTRACT_FIELDS
71+
or not all(_id(output_contract[key])
72+
for key in ("manifest_role", "candidate_role", "parser_revision"))
73+
or not _definition(output_contract["format_schema"])
74+
or output_contract["format_schema"] != context["output_contract"]["format_schema"]
75+
):
76+
raise ValueError("invalid generation output contract")
77+
roles = output_contract["required_input_roles"]
78+
if not isinstance(roles, list) or not all(_id(role) for role in roles):
79+
raise ValueError("invalid required input roles")
80+
if len(set(roles)) != len(roles):
81+
raise ValueError("duplicate required input role")
82+
delivery = context["delivered_context"]["artifact"]
83+
for key in ("task_materials", "allowed_feedback"):
84+
if not isinstance(context[key], list):
85+
raise ValueError("expected lists of authorized generation artifacts")
86+
allowed = set()
87+
for ref in [delivery, *context["task_materials"], *context["allowed_feedback"]]:
88+
validate_artifact_ref(ref)
89+
_logical_path(ref["location"])
90+
allowed.add(encode(ref))
91+
inputs = bind_inputs(deepcopy(context))
92+
if not isinstance(inputs, list):
93+
raise ValueError("expected generation inputs")
94+
by_role = _input_paths(inputs)
95+
if "generation_plan" in by_role:
96+
raise ValueError("the generation_plan input role is reserved for Harness sealing")
97+
used = set()
98+
for item in by_role.values():
99+
if item["source_namespace"] != "artifact":
100+
raise ValueError("generation inputs must use fixed artifacts")
101+
used.add(encode(item["source_ref"]))
102+
if not used <= allowed or encode(delivery) not in used:
103+
raise ValueError("expected the delivered context and only authorized fixed inputs")
104+
105+
if not isinstance(steps, list) or not steps:
106+
raise ValueError("expected generation command steps")
107+
outputs = []
108+
for step in steps:
109+
if step["expected_code_sha256"] is not None or not isinstance(step["outputs"], list):
110+
raise ValueError("expected ordinary generation command steps")
111+
outputs.extend(step["outputs"])
112+
paths = _output_paths(outputs)
113+
manifest_role = output_contract["manifest_role"]
114+
candidate_role = output_contract["candidate_role"]
115+
required = {output["role"]: output for output in outputs if output["required"]}
116+
if (
117+
manifest_role == candidate_role or manifest_role not in required
118+
or candidate_role not in required
119+
or required[candidate_role]["format_schema"] != output_contract["format_schema"]
120+
):
121+
raise ValueError("expected distinct required manifest and candidate outputs")
122+
_require_disjoint_paths([*paths.values(), *(item["materialized_path"] for item in inputs)])
123+
output_root = output_root_for(attempt["attempt_id"])
124+
_logical_path(output_root)
125+
if not isinstance(required_capabilities, list) or not all(
126+
_id(capability) for capability in required_capabilities
127+
):
128+
raise ValueError("invalid generation capabilities")
129+
plan = {
130+
"spec": {
131+
**attempt, "schema_version": 1, "generation_record_ref": None,
132+
"prepared_ref": None, "protocol_digest": None, "operation": None,
133+
"validation_scope": None, "candidate": None, "inputs": inputs,
134+
"steps": steps, "output_root": output_root,
135+
"required_capabilities": required_capabilities,
136+
},
137+
"manifest": {
138+
"schema_version": 1, "provider": provider, "output_contract": output_contract,
139+
"context_refs": [delivery], "attempt_id": attempt["attempt_id"],
140+
},
141+
}
142+
encode(plan)
143+
except (KeyError, TypeError, ValueError) as exc:
144+
raise PublicError(
145+
"InvalidExecutionSpec", stage="generation_plan",
146+
message="The generation plan could not be bound to the supplied context and attempt.",
147+
) from exc
148+
return deepcopy(plan)

‎tests/test_apps_generation_plan.py‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
"""Check the unsealed Apps plan handed to the Harness generation controller."""
2+
3+
from copy import deepcopy
4+
from hashlib import sha256
5+
from pathlib import Path
6+
7+
import pytest
8+
9+
from inferrsi.apps.api import CandidateProvider
10+
from inferrsi.contracts.json_v1 import decode, encode
11+
from inferrsi.harness.errors import PublicError
12+
from inferrsi.harness.execution_service import LocalCpuTarget, validate
13+
14+
_FIXTURES = Path(__file__).resolve().parents[1] / "docs/contracts/fixtures/t108-v4"
15+
_ATTEMPT_FIELDS = (
16+
"request_id", "run_id", "attempt_id", "owner", "target_request", "deadline_utc",
17+
"resources", "budget", "retry_of", "retry_reason",
18+
)
19+
20+
21+
def _artifact(name):
22+
body = name.encode()
23+
return {
24+
"location": f"stores/test/{name}", "sha256": sha256(body).hexdigest(),
25+
"media_type": "application/octet-stream", "size_bytes": len(body),
26+
}
27+
28+
29+
def _fixed_input(role, ref):
30+
return {
31+
"role": role, "source": ref["location"], "source_namespace": "artifact",
32+
"source_ref": ref, "snapshot_format": "file-v1", "expected_sha256": ref["sha256"],
33+
"materialized_path": f"inputs/{role}",
34+
}
35+
36+
37+
@pytest.fixture
38+
def generation():
39+
seed = decode((_FIXTURES / "execution-spec.json").read_bytes())
40+
prepared = decode((_FIXTURES / "prepared.json").read_bytes())
41+
context_ref, material, feedback = map(_artifact, ("context", "task", "feedback"))
42+
candidate_contract = prepared["candidate_contract"]
43+
return {
44+
"context": {
45+
"campaign_id": "campaign-1", "generation_id": "generation-1",
46+
"candidate": None, "parent": None, "task_materials": [material],
47+
"delivered_context": {
48+
"artifact": context_ref, "items": [], "selection": {},
49+
"omissions": [], "limitations": [],
50+
},
51+
"allowed_feedback": [feedback], "output_contract": candidate_contract,
52+
},
53+
"attempt": {field: seed[field] for field in _ATTEMPT_FIELDS},
54+
"provider": {"name": "cpu-example", "revision": "1"},
55+
"output_contract": {
56+
"manifest_role": "generation-manifest", "candidate_role": "candidate",
57+
"format_schema": candidate_contract["format_schema"],
58+
"required_input_roles": ["candidate"], "parser_revision": "1",
59+
},
60+
"inputs": [
61+
_fixed_input("knowledge", context_ref),
62+
_fixed_input("task", material),
63+
_fixed_input("feedback", feedback),
64+
],
65+
"steps": [{
66+
"step_id": "generate", "runtime_role": "local-python",
67+
"entrypoint_kind": "python_module", "entrypoint": "fixture_generation",
68+
"entrypoint_revision": "1", "expected_code_sha256": None,
69+
"args": ["inputs/knowledge", "output/candidate.txt"], "cwd": ".",
70+
"timeout_seconds": None,
71+
"outputs": [
72+
{"role": "generation-manifest", "path": "output/manifest.json", "required": True,
73+
"media_type": "application/json",
74+
"format_schema": {"name": "fixture-generation-manifest", "revision": "1"}},
75+
{"role": "candidate", "path": "output/candidate.txt", "required": True,
76+
"media_type": "text/plain", "format_schema": candidate_contract["format_schema"]},
77+
],
78+
}],
79+
"required_capabilities": ["local-process-tree"],
80+
}
81+
82+
83+
def _provider(generation, bind_inputs=None):
84+
return CandidateProvider(
85+
provider=generation["provider"], output_contract=generation["output_contract"],
86+
steps=generation["steps"], required_capabilities=generation["required_capabilities"],
87+
bind_inputs=bind_inputs or (lambda context: generation["inputs"]),
88+
output_root_for=lambda attempt_id: f"generation/{attempt_id}",
89+
)
90+
91+
92+
def test_plan_preserves_attempt_and_keeps_manifest_separate(generation, tmp_path):
93+
original = deepcopy(generation)
94+
provider = _provider(generation)
95+
result = provider.plan_generation(generation["context"], generation["attempt"])
96+
spec, manifest = result["spec"], result["manifest"]
97+
98+
assert {key: spec[key] for key in _ATTEMPT_FIELDS} == generation["attempt"]
99+
assert all(spec[field] is None for field in (
100+
"generation_record_ref", "prepared_ref", "protocol_digest", "operation",
101+
"validation_scope", "candidate",
102+
))
103+
assert spec["inputs"] == generation["inputs"]
104+
assert spec["steps"] == generation["steps"]
105+
assert spec["output_root"] == f"generation/{generation['attempt']['attempt_id']}"
106+
assert manifest == {
107+
"schema_version": 1, "provider": generation["provider"],
108+
"output_contract": generation["output_contract"],
109+
"context_refs": [generation["context"]["delivered_context"]["artifact"]],
110+
"attempt_id": generation["attempt"]["attempt_id"],
111+
}
112+
target_request = spec["target_request"]
113+
validate(spec, LocalCpuTarget(
114+
target_request["target_id"], target_request["config_revision"], tmp_path,
115+
))
116+
assert encode(provider.plan_generation(generation["context"], generation["attempt"])) == encode(
117+
result,
118+
)
119+
spec["inputs"][0]["source_ref"]["location"] = "other"
120+
manifest["output_contract"]["required_input_roles"].clear()
121+
assert generation == original
122+
123+
124+
def test_provider_binds_each_context_with_the_same_worker_configuration(generation):
125+
def bind_context(context):
126+
return [
127+
_fixed_input("knowledge", context["delivered_context"]["artifact"]),
128+
_fixed_input("task", context["task_materials"][0]),
129+
_fixed_input("feedback", context["allowed_feedback"][0]),
130+
]
131+
132+
context, attempt = generation["context"], generation["attempt"]
133+
provider = _provider(generation, bind_context)
134+
first = provider.plan_generation(context, attempt)
135+
next_context = deepcopy(context)
136+
next_context["delivered_context"]["artifact"] = _artifact("another-context")
137+
next_context["allowed_feedback"] = [_artifact("another-feedback")]
138+
next_attempt = {**attempt, "attempt_id": "next-attempt", "request_id": "next-attempt"}
139+
generation["steps"][0]["entrypoint"] = "altered-after-configuration"
140+
141+
second = provider.plan_generation(next_context, next_attempt)
142+
143+
assert second["spec"]["steps"] == first["spec"]["steps"]
144+
assert second["spec"]["output_root"] == "generation/next-attempt"
145+
assert second["manifest"]["attempt_id"] == "next-attempt"
146+
assert second["manifest"]["context_refs"] == [next_context["delivered_context"]["artifact"]]
147+
assert second["spec"]["inputs"] == bind_context(next_context)
148+
149+
150+
@pytest.mark.parametrize(("path", "value"), (
151+
pytest.param(("context", "delivered_context"), {"content": b"unsealed"}, id="draft"),
152+
pytest.param(("inputs",), [], id="missing-delivery"),
153+
pytest.param(("inputs", 2), _fixed_input("feedback", _artifact("foreign")),
154+
id="foreign-feedback"),
155+
pytest.param(("inputs", 0, "role"), "generation_plan", id="reserved-role"),
156+
pytest.param(("steps", 0, "outputs"), [], id="missing-outputs"),
157+
pytest.param(("steps", 0, "outputs", 1, "required"), False, id="optional-candidate"),
158+
pytest.param(("steps", 0, "outputs", 1, "format_schema"), {"name": "other", "revision": "1"},
159+
id="wrong-format"),
160+
pytest.param(("steps", 0, "expected_code_sha256"), "0" * 64, id="worker-digest"),
161+
pytest.param(("attempt", "request_id"), "different-attempt", id="new-request-id"),
162+
))
163+
def test_plan_rejects_invalid_handoff(generation, path, value):
164+
target = generation
165+
for key in path[:-1]:
166+
target = target[key]
167+
target[path[-1]] = value
168+
169+
with pytest.raises(PublicError) as caught:
170+
_provider(generation).plan_generation(generation["context"], generation["attempt"])
171+
assert (caught.value.code, caught.value.stage) == ("InvalidExecutionSpec", "generation_plan")

0 commit comments

Comments
 (0)