From 82b2960eb3e66352df1a666c75d661ad96b45001 Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Tue, 30 Jun 2026 12:42:55 -0700 Subject: [PATCH] feat(canonicalization): use RFC 8785 (JCS) for sign/verify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the ad-hoc json.dumps(sort_keys=True, ensure_ascii=True) canonicalization in _canonical_bytes with the RFC 8785-conformant rfc8785 library. Both sign_record and verify_record share _canonical_bytes, so this single change covers signing and verification. The spec (spec/trace-v0.1.md §3.2.2) normatively mandates RFC 8785 (JCS). json.dumps was not JCS-compliant: it escapes non-ASCII as \uXXXX instead of raw UTF-8, zero-pads number exponents (1e-07 vs JCS 1e-7), and sorts keys by Unicode code point rather than UTF-16 code unit. These divergences broke cross-implementation verification and allowed signature-preserving mutation. Also fixes the doc snippets in docs/verification.md and docs/tutorials/signing-your-first-trust-record.md that showed json.dumps as the canonicalization, so docs match the code. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../signing-your-first-trust-record.md | 7 +- docs/verification.md | 7 +- pyproject.toml | 1 + src/agentrust_trace/sign.py | 15 +++- tests/test_sign.py | 76 +++++++++++++++++++ 5 files changed, 99 insertions(+), 7 deletions(-) diff --git a/docs/tutorials/signing-your-first-trust-record.md b/docs/tutorials/signing-your-first-trust-record.md index 1278696..896c555 100644 --- a/docs/tutorials/signing-your-first-trust-record.md +++ b/docs/tutorials/signing-your-first-trust-record.md @@ -112,11 +112,12 @@ The signature covers every field in the record except `signature` itself. `cnf.j The library signs the canonical byte representation of the record with the `signature` field removed. Canonicalization follows RFC 8785 JSON Canonicalization Scheme (JCS): -- Object keys sorted in Unicode code-point order (ascending) +- Object keys sorted in UTF-16 code-unit order (ascending) - No whitespace between tokens -- Numbers serialized in IEEE 754 double-precision shortest form +- Numbers serialized in IEEE 754 double-precision shortest round-trip form (RFC 8785 §3.2.2.3) +- Strings emitted as raw UTF-8, escaping only the characters RFC 8259 §7 requires -`json.dumps(record, sort_keys=True)` produces a different byte sequence than JCS for Unicode keys whose sort order differs between Python and Unicode code-point order. The library uses `_canonical_bytes()` internally, which calls `json.dumps(..., sort_keys=True, separators=(",", ":"), ensure_ascii=True)`. For plain ASCII keys this matches JCS. Records with non-ASCII keys require a dedicated JCS library; use ASCII-only field names to stay portable. +`_canonical_bytes()` is implemented with the RFC 8785-conformant [`rfc8785`](https://pypi.org/project/rfc8785/) library. `json.dumps(record, sort_keys=True, ensure_ascii=True)` is **not** a substitute: it escapes non-ASCII characters as `\uXXXX`, zero-pads number exponents (`1e-07` vs JCS `1e-7`), and sorts by Unicode code point rather than UTF-16 code unit. Any of these would break cross-implementation verification and allow signature-preserving mutation, so the library is used in both signing and verification. The spec (section 3.2.2) requires JCS canonical form. Do not reimplement this by hand. diff --git a/docs/verification.md b/docs/verification.md index 580b8a3..40d5127 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -12,14 +12,17 @@ A TRACE Trust Record is a signed JSON object. The `signature` field contains a b ```python import json, base64 +import rfc8785 # RFC 8785 (JCS) canonicalization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey record = json.load(open("session.trace.json")) sig_bytes = base64.urlsafe_b64decode(record["signature"] + "==") -payload = {k: v for k, v in record.items() if k not in ("signature", "cnf")} -payload_bytes = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() +payload = {k: v for k, v in record.items() if k != "signature"} +payload_bytes = rfc8785.dumps(payload) # JCS canonical bytes, NOT json.dumps ``` +The pre-image is the RFC 8785 (JCS) canonical form of the record with `signature` removed. `json.dumps(sort_keys=True)` is **not** JCS-conformant — it diverges for non-ASCII strings and IEEE 754 numbers — so use a JCS library (the spec mandates this in §3.2.2). + ### Step 2 — Resolve the public key The `cnf.jwk` field embeds the public key. For TEE-issued records, this key is TEE-bound — its private half never leaves the measured enclave. diff --git a/pyproject.toml b/pyproject.toml index eb63484..dbb1bc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "pydantic>=2.0", "jsonschema>=4.20", "cryptography>=42.0", + "rfc8785>=0.1.2", ] [project.optional-dependencies] diff --git a/src/agentrust_trace/sign.py b/src/agentrust_trace/sign.py index bed10e4..858c7cd 100644 --- a/src/agentrust_trace/sign.py +++ b/src/agentrust_trace/sign.py @@ -10,11 +10,11 @@ import base64 import binascii -import json import os import warnings from typing import Any +import rfc8785 from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey @@ -59,7 +59,18 @@ def key_to_jwk(key: Ed25519PrivateKey) -> dict[str, str]: def _canonical_bytes(d: dict[str, Any]) -> bytes: - return json.dumps(d, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + """Return the RFC 8785 (JCS) canonical UTF-8 byte sequence for *d*. + + This is the signature pre-image mandated by spec/trace-v0.1.md §3.2.2. JCS + sorts object keys by UTF-16 code unit, serializes numbers per the + ECMAScript Number-to-String / RFC 8785 §3.2.2.3 shortest round-trip form, + escapes only the characters required by RFC 8259 §7, and emits non-ASCII + characters as raw UTF-8 (not ``\\uXXXX`` escapes). A plain + ``json.dumps(sort_keys=True)`` diverges from JCS for non-ASCII strings and + for IEEE 754 number formatting, which would break cross-implementation + verification, so a conformant library is used instead. + """ + return rfc8785.dumps(d) def _b64url_decode(value: str, *, field: str) -> bytes: diff --git a/tests/test_sign.py b/tests/test_sign.py index 341692a..d8366ab 100644 --- a/tests/test_sign.py +++ b/tests/test_sign.py @@ -1,9 +1,11 @@ """Tests for agentrust_trace.sign.""" import base64 +import json import time import pytest +import rfc8785 from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey @@ -219,3 +221,77 @@ def test_verify_record_rejects_malformed_signature(): record["signature"] = "!!!not base64!!!" with pytest.raises(ValueError, match="base64url"): verify_record(record, key_to_jwk(key)) + + +# --- RFC 8785 (JCS) canonicalization ---------------------------------------- + + +def test_canonical_bytes_sorts_keys_no_whitespace(): + """JCS sorts object keys and emits no inter-token whitespace.""" + assert _canonical_bytes({"b": 1, "a": 2, "c": 3}) == b'{"a":2,"b":1,"c":3}' + + +def test_canonical_bytes_known_answer_non_ascii(): + """Known-answer vector: non-ASCII strings are raw UTF-8, NOT \\uXXXX escapes. + + This is the headline divergence from ``json.dumps(..., ensure_ascii=True)``, + which the old implementation used. A regression to json.dumps would emit the + escaped form on the right and fail this literal-bytes assertion. + """ + obj = {"msg": "hüllo", "id": "café"} + expected = b'{"id":"caf\xc3\xa9","msg":"h\xc3\xbcllo"}' + assert _canonical_bytes(obj) == expected + # The discarded json.dumps form would have escaped the non-ASCII code points. + assert json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() == ( + b'{"id":"caf\\u00e9","msg":"h\\u00fcllo"}' + ) + assert _canonical_bytes(obj) != json.dumps( + obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + + +def test_canonical_bytes_known_answer_numbers(): + """Known-answer vector for JCS / RFC 8785 §3.2.2.3 number serialization. + + ``1e-7`` is the RFC 8785 shortest round-trip form; Python's ``json.dumps`` + emits ``1e-07`` (zero-padded exponent), so this literal assertion catches a + regression to json.dumps for number formatting. + """ + assert _canonical_bytes({"n": 1e-7}) == b'{"n":1e-7}' + assert json.dumps(1e-7) == "1e-07" # the non-conformant form we moved away from + + +def test_canonical_bytes_matches_reference_library(): + """Cross-check the full record canonicalization against the rfc8785 reference.""" + record = _minimal_record() + record["model"]["note"] = "résumé" # non-ASCII to exercise the divergence + assert _canonical_bytes(record) == rfc8785.dumps(record) + + +def test_jcs_distinguishes_unicode_key_order_from_json_dumps(): + """JCS sorts keys by UTF-16 code unit; this differs from naive byte sorting. + + Astral-plane code points (here U+1F600, encoded as a surrogate pair in + UTF-16) sort AFTER the BMP key ``z`` under UTF-16 code-unit ordering. Python + ``json.dumps(sort_keys=True)`` sorts by Unicode code point, placing the + astral key (U+1F600) BEFORE ``z`` (U+007A). The two schemes therefore + produce different byte sequences for the same object, even though both claim + to "sort keys". + """ + obj = {"z": 1, "\U0001f600": 2} + jcs = _canonical_bytes(obj) + jdump = json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + # JCS (UTF-16 code unit): high surrogate 0xD83D > 0x007A, so "z" comes first. + assert jcs == b'{"z":1,"\xf0\x9f\x98\x80":2}' + # json.dumps (code point): 0x1F600 > 0x007A as a scalar, so order matches here, + # but the emoji is escaped as a surrogate pair, diverging in bytes regardless. + assert jcs != jdump + + +def test_round_trip_with_non_ascii_payload(): + """End-to-end: signing and verifying a record carrying non-ASCII data.""" + key = generate_key() + record = _fresh_record() + record["model"]["note"] = "modèle français \U0001f916" + signed = sign_record(record, key) + verify_record(signed, key_to_jwk(key)) # must not raise