@@ -34,11 +34,50 @@ def edit_observation(packet, change):
3434 edit (packet , "measurements.json" , lambda m : m ["observations" ].__setitem__ (0 , observation ))
3535
3636
37+ def test_pinned_producer_augments_measurements_after_saving_sidecars (packet ):
38+ measurements = json .loads ((packet .parent / "measurements.json" ).read_bytes ())
39+ augmented = []
40+ for observation in measurements ["observations" ]:
41+ name = observation ["case" ]
42+ group = name .rsplit ("-" , 1 )[0 ]
43+ sidecar = json .loads (
44+ (packet .parent / f"run/{ group } /{ name } /observation.json" ).read_bytes ()
45+ )
46+ if group in ("ttft-4096-r0" , "prefill-65536-r0" ):
47+ assert "http_prefill_tokens_per_second" not in sidecar ["analysis" ]
48+ expected = {
49+ ** sidecar ,
50+ "analysis" : {
51+ ** sidecar ["analysis" ],
52+ "http_prefill_tokens_per_second" : (
53+ (4096 if group == "ttft-4096-r0" else 65536 ) * 1e9 / 100_000_000
54+ ),
55+ },
56+ }
57+ assert observation == expected
58+ augmented .append (name )
59+ else :
60+ assert observation == sidecar
61+ assert len (augmented ) == 2
62+ receipt = adapt (packet )
63+ assert receipt ["metrics" ] == {"delivery_ttft_ns" : 100_000_000 }
64+
65+
3766def test_complete_raw_packet_recomputes_metric_and_preserves_originals (packet , tmp_path ):
67+ def change_summaries (measurements ):
68+ measurements .update (summary = {"delivery_ttft_ns" : - 999 })
69+ observation = next (
70+ o for o in measurements ["observations" ] if o ["case" ] == "ttft-512-r0-0"
71+ )
72+ observation ["analysis" ]["delivery_ttft_ns" ] = - 999
73+
74+ edit (packet , "measurements.json" , change_summaries )
75+ edit (
76+ packet ,
77+ "run/ttft-512-r0/ttft-512-r0-0/observation.json" ,
78+ lambda o : o ["analysis" ].update (delivery_ttft_ns = - 999 ),
79+ )
3880 before = {p : p .read_bytes () for p in packet .parent .rglob ("*" ) if p .is_file ()}
39- edit (packet , "measurements.json" , lambda m : m .update (summary = {"delivery_ttft_ns" : - 999 }))
40- before [packet ] = packet .read_bytes ()
41- before [packet .parent / "measurements.json" ] = (packet .parent / "measurements.json" ).read_bytes ()
4281 receipt = adapt (packet )
4382 assert receipt ["metrics" ] == {"delivery_ttft_ns" : 100_000_000 }
4483 assert receipt ["synthetic" ] is True
@@ -56,6 +95,61 @@ def test_complete_raw_packet_recomputes_metric_and_preserves_originals(packet, t
5695 assert json .loads (output .read_bytes ())["identities" ]["manifest_sha256" ]
5796
5897
98+ def test_identical_unaugmented_observations_remain_supported (packet ):
99+ def remove_derived (measurements ):
100+ for observation in measurements ["observations" ]:
101+ observation ["analysis" ].pop ("http_prefill_tokens_per_second" , None )
102+
103+ edit (packet , "measurements.json" , remove_derived )
104+ assert adapt (packet )["metrics" ] == {"delivery_ttft_ns" : 100_000_000 }
105+
106+
107+ @pytest .mark .parametrize ("side" , ["sidecar" , "measurements" ])
108+ @pytest .mark .parametrize (
109+ "mutation" , ["timestamp" , "status" , "request" , "analysis" , "extra_analysis" ]
110+ )
111+ def test_divergent_observation_rejects_even_with_refreshed_digest (packet , tmp_path , side , mutation ):
112+ def change (observation ):
113+ if mutation == "timestamp" :
114+ observation ["events" ][0 ]["elapsed_ns" ] += 1
115+ elif mutation == "status" :
116+ observation ["status" ] = 201
117+ elif mutation == "request" :
118+ observation ["request_sha256" ] = "0" * 64
119+ elif mutation == "analysis" :
120+ observation ["analysis" ]["delivery_ttft_ns" ] += 1
121+ else :
122+ observation ["analysis" ]["unrecognized_derived_value" ] = 123
123+
124+ if side == "sidecar" :
125+ edit (packet , "run/ttft-4096-r0/ttft-4096-r0-0/observation.json" , change )
126+ else :
127+ edit (
128+ packet ,
129+ "measurements.json" ,
130+ lambda m : change (next (o for o in m ["observations" ] if o ["case" ] == "ttft-4096-r0-0" )),
131+ )
132+ output = tmp_path / "rejected.json"
133+ assert (
134+ main (["measure" , "--packet" , str (packet ), "--peer" , str (packet ), "--output" , str (output )])
135+ == 2
136+ )
137+ rejected = json .loads (output .read_bytes ())
138+ assert rejected ["error" ] == "observation sidecar differs"
139+ assert rejected ["metrics" ] == {}
140+
141+
142+ @pytest .mark .parametrize ("case" , ["warmup-0" , "ttft-4096-r0-0" ])
143+ def test_derived_analysis_exception_rejects_wrong_group_or_value (packet , case ):
144+ def change (measurements ):
145+ observation = next (o for o in measurements ["observations" ] if o ["case" ] == case )
146+ observation ["analysis" ]["http_prefill_tokens_per_second" ] = - 1
147+
148+ edit (packet , "measurements.json" , change )
149+ with pytest .raises (ValueError , match = "observation sidecar differs" ):
150+ adapt (packet )
151+
152+
59153@pytest .mark .parametrize (
60154 "mutation" ,
61155 [
0 commit comments