Skip to content

Commit 5a2d16e

Browse files
committed
fix: accept Flare post-save derived analysis
Match the pinned producer augmentation without relaxing original observation integrity or raw metric recomputation. Exercise its write order and reject divergent fields even with refreshed packet digests. Agent-assisted-by: Codex (author) Signed-off-by: GUOGUO <xwdong1998@163.com>
1 parent a4a0fbd commit 5a2d16e

4 files changed

Lines changed: 145 additions & 10 deletions

File tree

‎docs/flare-qualification.md‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,16 @@ it does not load a tokenizer or prove that a rendered prompt tokenizes correctly
6363

6464
Each observation must match its sidecar and raw SSE payloads, successful HTTP
6565
completion, terminal DONE, prompt/output/usage counts and sent request identity.
66+
Pinned Flare saves each sidecar before `run` adds
67+
`analysis.http_prefill_tokens_per_second` to measurements for `ttft-4096` and
68+
`prefill-65536`. The adapter accepts that one addition only with the pinned
69+
formula (`prompt_token_count * 1e9 / analysis.delivery_ttft_ns`); every original
70+
field, including the rest of analysis, must still match. Identical unaugmented
71+
copies remain supported. The adapter does not modify either input copy.
6672
Coalesced groups must agree with any recorded grouping analysis. Summary TTFT
67-
numbers are not metric inputs. Explicit failed qualification/group flags and
68-
stream errors remain rejections even if other raw checks pass.
73+
and derived throughput numbers are not metric inputs. Explicit failed
74+
qualification/group flags and stream errors remain rejections even if other raw
75+
checks pass.
6976

7077
The native `prefill_chunk_size` label must match the preparation report. Selected
7178
raw snapshots must contain unlabelled integer samples for the scheduler gauges,
@@ -146,12 +153,14 @@ pytest tests/test_flare_adapter.py
146153

147154
The fixture writer saves a complete one-repetition packet, including raw SSE,
148155
coalesced token events, a prefix-hit case and before/after resource snapshots.
156+
It follows the pinned producer's sidecar-before-derived-analysis write order.
149157
It labels every packet `synthetic=true`; no tokenizer, model, server or device
150158
is used. Synthetic delivery timings and model/package placeholders are parser
151159
test inputs, not measured service results.
152160

153161
Tests cover raw recomputation and unchanged originals; missing raw/group/request
154162
records; wrong clock declarations; invented coalesced-token timing; capacity,
155163
counter and resource mismatches; stream failures and digest drift; pair scope;
156-
and real local runner execution of both adapter commands. Fixture acceptance
157-
does not establish service quality, a performance gain or agent efficacy.
164+
sidecar/measurements divergence even with refreshed digests; and real local
165+
runner execution of both adapter commands. Fixture acceptance does not establish
166+
service quality, a performance gain or agent efficacy.

‎src/inferrsi/model/validators/flare.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,28 @@ def adapt(packet_path: Path) -> dict:
330330
results = []
331331
for name in group["cases"]:
332332
observation = json.loads(files[f"{root}/{name}/observation.json"])
333-
_require(observation in observations, "observation sidecar differs")
333+
expected = observation
334+
analysis = observation.get("analysis", {})
335+
ttft = analysis.get("delivery_ttft_ns")
336+
if (
337+
group["kind"] in ("ttft-4096", "prefill-65536")
338+
and ttft
339+
and "http_prefill_tokens_per_second" not in analysis
340+
):
341+
# Pinned Flare adds this derived value only after saving the sidecar.
342+
expected = {
343+
**observation,
344+
"analysis": {
345+
**analysis,
346+
"http_prefill_tokens_per_second": (
347+
len(cases[name]["prompt_tokens"]) * 1e9 / ttft
348+
),
349+
},
350+
}
351+
_require(
352+
observation in observations or expected in observations,
353+
"observation sidecar differs",
354+
)
334355
request = json.loads(files[f"{root}/{name}/request.json"])
335356
expected_request = dict(cases[name]["request"])
336357
expected_request.update(

‎tests/fixtures/flare_packet.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ def add(name, kind, batch, repetition, measured):
114114
for group in groups:
115115
folder = root / "run" / group["id"]
116116
folder.mkdir(parents=True)
117+
observations = []
117118
reused = 0
118119
for identifier in group["cases"]:
119120
case = next(c for c in cases if c["id"] == identifier)
@@ -179,21 +180,22 @@ def add(name, kind, batch, repetition, measured):
179180
"events": events,
180181
"analysis": {
181182
"passed": True,
183+
"delivery_ttft_ns": ttft_ns,
182184
"token_groups": token_groups,
183185
"delivery_itl_exact": len(output) == 1,
184186
"delivery_itl_ns": [],
185187
},
186188
}
187189
write_json(local / "request.json", request)
188-
write_json(local / "observation.json", observation)
189190
raw = b"".join(
190191
b"data: "
191192
+ (e["data"].encode() if isinstance(e["data"], str) else encoded(e["data"]))
192193
+ b"\n\n"
193194
for e in events
194195
)
195196
(local / "response.sse").write_bytes(raw)
196-
measurements["observations"].append(observation)
197+
write_json(local / "observation.json", observation)
198+
observations.append(observation)
197199
values = {
198200
"flare_active_requests": 0,
199201
"flare_free_request_slots": 4,
@@ -220,6 +222,15 @@ def add(name, kind, batch, repetition, measured):
220222
names.append(name)
221223
snapshots[group["id"]] = names
222224
measurements["groups"].append({"id": group["id"], "passed": True})
225+
# Pinned Flare augments in-memory analysis after saving observation sidecars.
226+
for observation in observations:
227+
ttft = observation["analysis"]["delivery_ttft_ns"]
228+
if ttft and group["kind"] in ("ttft-4096", "prefill-65536"):
229+
case = next(c for c in cases if c["id"] == observation["case"])
230+
observation["analysis"]["http_prefill_tokens_per_second"] = (
231+
len(case["prompt_tokens"]) * 1e9 / ttft
232+
)
233+
measurements["observations"].extend(observations)
223234
write_json(root / "measurements.json", measurements)
224235
manifest = {
225236
"schema": SCHEMA,

‎tests/test_flare_adapter.py‎

Lines changed: 97 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,50 @@ def edit_observation(packet, change):
3434
edit(packet, "measurements.json", lambda m: m["observations"].__setitem__(0, observation))
3535

3636

37+
def test_pinned_producer_augments_measurements_after_saving_sidecars(packet):
38+
measurements = json.loads((packet.parent / "measurements.json").read_bytes())
39+
augmented = []
40+
for observation in measurements["observations"]:
41+
name = observation["case"]
42+
group = name.rsplit("-", 1)[0]
43+
sidecar = json.loads(
44+
(packet.parent / f"run/{group}/{name}/observation.json").read_bytes()
45+
)
46+
if group in ("ttft-4096-r0", "prefill-65536-r0"):
47+
assert "http_prefill_tokens_per_second" not in sidecar["analysis"]
48+
expected = {
49+
**sidecar,
50+
"analysis": {
51+
**sidecar["analysis"],
52+
"http_prefill_tokens_per_second": (
53+
(4096 if group == "ttft-4096-r0" else 65536) * 1e9 / 100_000_000
54+
),
55+
},
56+
}
57+
assert observation == expected
58+
augmented.append(name)
59+
else:
60+
assert observation == sidecar
61+
assert len(augmented) == 2
62+
receipt = adapt(packet)
63+
assert receipt["metrics"] == {"delivery_ttft_ns": 100_000_000}
64+
65+
3766
def test_complete_raw_packet_recomputes_metric_and_preserves_originals(packet, tmp_path):
67+
def change_summaries(measurements):
68+
measurements.update(summary={"delivery_ttft_ns": -999})
69+
observation = next(
70+
o for o in measurements["observations"] if o["case"] == "ttft-512-r0-0"
71+
)
72+
observation["analysis"]["delivery_ttft_ns"] = -999
73+
74+
edit(packet, "measurements.json", change_summaries)
75+
edit(
76+
packet,
77+
"run/ttft-512-r0/ttft-512-r0-0/observation.json",
78+
lambda o: o["analysis"].update(delivery_ttft_ns=-999),
79+
)
3880
before = {p: p.read_bytes() for p in packet.parent.rglob("*") if p.is_file()}
39-
edit(packet, "measurements.json", lambda m: m.update(summary={"delivery_ttft_ns": -999}))
40-
before[packet] = packet.read_bytes()
41-
before[packet.parent / "measurements.json"] = (packet.parent / "measurements.json").read_bytes()
4281
receipt = adapt(packet)
4382
assert receipt["metrics"] == {"delivery_ttft_ns": 100_000_000}
4483
assert receipt["synthetic"] is True
@@ -56,6 +95,61 @@ def test_complete_raw_packet_recomputes_metric_and_preserves_originals(packet, t
5695
assert json.loads(output.read_bytes())["identities"]["manifest_sha256"]
5796

5897

98+
def test_identical_unaugmented_observations_remain_supported(packet):
99+
def remove_derived(measurements):
100+
for observation in measurements["observations"]:
101+
observation["analysis"].pop("http_prefill_tokens_per_second", None)
102+
103+
edit(packet, "measurements.json", remove_derived)
104+
assert adapt(packet)["metrics"] == {"delivery_ttft_ns": 100_000_000}
105+
106+
107+
@pytest.mark.parametrize("side", ["sidecar", "measurements"])
108+
@pytest.mark.parametrize(
109+
"mutation", ["timestamp", "status", "request", "analysis", "extra_analysis"]
110+
)
111+
def test_divergent_observation_rejects_even_with_refreshed_digest(packet, tmp_path, side, mutation):
112+
def change(observation):
113+
if mutation == "timestamp":
114+
observation["events"][0]["elapsed_ns"] += 1
115+
elif mutation == "status":
116+
observation["status"] = 201
117+
elif mutation == "request":
118+
observation["request_sha256"] = "0" * 64
119+
elif mutation == "analysis":
120+
observation["analysis"]["delivery_ttft_ns"] += 1
121+
else:
122+
observation["analysis"]["unrecognized_derived_value"] = 123
123+
124+
if side == "sidecar":
125+
edit(packet, "run/ttft-4096-r0/ttft-4096-r0-0/observation.json", change)
126+
else:
127+
edit(
128+
packet,
129+
"measurements.json",
130+
lambda m: change(next(o for o in m["observations"] if o["case"] == "ttft-4096-r0-0")),
131+
)
132+
output = tmp_path / "rejected.json"
133+
assert (
134+
main(["measure", "--packet", str(packet), "--peer", str(packet), "--output", str(output)])
135+
== 2
136+
)
137+
rejected = json.loads(output.read_bytes())
138+
assert rejected["error"] == "observation sidecar differs"
139+
assert rejected["metrics"] == {}
140+
141+
142+
@pytest.mark.parametrize("case", ["warmup-0", "ttft-4096-r0-0"])
143+
def test_derived_analysis_exception_rejects_wrong_group_or_value(packet, case):
144+
def change(measurements):
145+
observation = next(o for o in measurements["observations"] if o["case"] == case)
146+
observation["analysis"]["http_prefill_tokens_per_second"] = -1
147+
148+
edit(packet, "measurements.json", change)
149+
with pytest.raises(ValueError, match="observation sidecar differs"):
150+
adapt(packet)
151+
152+
59153
@pytest.mark.parametrize(
60154
"mutation",
61155
[

0 commit comments

Comments
 (0)