feat(canonicalization): use RFC 8785 (JCS) for sign/verify

docs/tutorials/signing-your-first-trust-record.md

@@ -112,11 +112,12 @@ The signature covers every field in the record except `signature` itself. `cnf.j
 
 The library signs the canonical byte representation of the record with the `signature` field removed. Canonicalization follows RFC 8785 JSON Canonicalization Scheme (JCS):
 
-- Object keys sorted in Unicode code-point order (ascending)
+- Object keys sorted in UTF-16 code-unit order (ascending)
 - No whitespace between tokens
-- Numbers serialized in IEEE 754 double-precision shortest form
+- Numbers serialized in IEEE 754 double-precision shortest round-trip form (RFC 8785 §3.2.2.3)
+- Strings emitted as raw UTF-8, escaping only the characters RFC 8259 §7 requires
 
-`json.dumps(record, sort_keys=True)` produces a different byte sequence than JCS for Unicode keys whose sort order differs between Python and Unicode code-point order. The library uses `_canonical_bytes()` internally, which calls `json.dumps(..., sort_keys=True, separators=(",", ":"), ensure_ascii=True)`. For plain ASCII keys this matches JCS. Records with non-ASCII keys require a dedicated JCS library; use ASCII-only field names to stay portable.
+`_canonical_bytes()` is implemented with the RFC 8785-conformant [`rfc8785`](https://pypi.org/project/rfc8785/) library. `json.dumps(record, sort_keys=True, ensure_ascii=True)` is **not** a substitute: it escapes non-ASCII characters as `\uXXXX`, zero-pads number exponents (`1e-07` vs JCS `1e-7`), and sorts by Unicode code point rather than UTF-16 code unit. Any of these would break cross-implementation verification and allow signature-preserving mutation, so the library is used in both signing and verification.
 
 The spec (section 3.2.2) requires JCS canonical form. Do not reimplement this by hand.
 

docs/verification.md

@@ -12,14 +12,17 @@ A TRACE Trust Record is a signed JSON object. The `signature` field contains a b
 
 ```python
 import json, base64
+import rfc8785  # RFC 8785 (JCS) canonicalization
 from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
 
 record = json.load(open("session.trace.json"))
 sig_bytes = base64.urlsafe_b64decode(record["signature"] + "==")
-payload = {k: v for k, v in record.items() if k not in ("signature", "cnf")}
-payload_bytes = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
+payload = {k: v for k, v in record.items() if k != "signature"}
+payload_bytes = rfc8785.dumps(payload)  # JCS canonical bytes, NOT json.dumps
 ```
 
+The pre-image is the RFC 8785 (JCS) canonical form of the record with `signature` removed. `json.dumps(sort_keys=True)` is **not** JCS-conformant — it diverges for non-ASCII strings and IEEE 754 numbers — so use a JCS library (the spec mandates this in §3.2.2).
+
 ### Step 2 — Resolve the public key
 
 The `cnf.jwk` field embeds the public key. For TEE-issued records, this key is TEE-bound — its private half never leaves the measured enclave.

pyproject.toml

@@ -25,6 +25,7 @@ dependencies = [
     "pydantic>=2.0",
     "jsonschema>=4.20",
     "cryptography>=42.0",
+    "rfc8785>=0.1.2",
 ]
 
 [project.optional-dependencies]

src/agentrust_trace/sign.py

@@ -10,11 +10,11 @@
 
 import base64
 import binascii
-import json
 import os
 import warnings
 from typing import Any
 
+import rfc8785
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
 
@@ -59,7 +59,18 @@ def key_to_jwk(key: Ed25519PrivateKey) -> dict[str, str]:
 
 
 def _canonical_bytes(d: dict[str, Any]) -> bytes:
-    return json.dumps(d, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()
+    """Return the RFC 8785 (JCS) canonical UTF-8 byte sequence for *d*.
+
+    This is the signature pre-image mandated by spec/trace-v0.1.md §3.2.2. JCS
+    sorts object keys by UTF-16 code unit, serializes numbers per the
+    ECMAScript Number-to-String / RFC 8785 §3.2.2.3 shortest round-trip form,
+    escapes only the characters required by RFC 8259 §7, and emits non-ASCII
+    characters as raw UTF-8 (not ``\\uXXXX`` escapes). A plain
+    ``json.dumps(sort_keys=True)`` diverges from JCS for non-ASCII strings and
+    for IEEE 754 number formatting, which would break cross-implementation
+    verification, so a conformant library is used instead.
+    """
+    return rfc8785.dumps(d)
 
 
 def _b64url_decode(value: str, *, field: str) -> bytes:

tests/test_sign.py

@@ -1,9 +1,11 @@
 """Tests for agentrust_trace.sign."""
 
 import base64
+import json
 import time
 
 import pytest
+import rfc8785
 from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
 
@@ -219,3 +221,77 @@ def test_verify_record_rejects_malformed_signature():
     record["signature"] = "!!!not base64!!!"
     with pytest.raises(ValueError, match="base64url"):
         verify_record(record, key_to_jwk(key))
+
+
+# --- RFC 8785 (JCS) canonicalization ----------------------------------------
+
+
+def test_canonical_bytes_sorts_keys_no_whitespace():
+    """JCS sorts object keys and emits no inter-token whitespace."""
+    assert _canonical_bytes({"b": 1, "a": 2, "c": 3}) == b'{"a":2,"b":1,"c":3}'
+
+
+def test_canonical_bytes_known_answer_non_ascii():
+    """Known-answer vector: non-ASCII strings are raw UTF-8, NOT \\uXXXX escapes.
+
+    This is the headline divergence from ``json.dumps(..., ensure_ascii=True)``,
+    which the old implementation used. A regression to json.dumps would emit the
+    escaped form on the right and fail this literal-bytes assertion.
+    """
+    obj = {"msg": "hüllo", "id": "café"}
+    expected = b'{"id":"caf\xc3\xa9","msg":"h\xc3\xbcllo"}'
+    assert _canonical_bytes(obj) == expected
+    # The discarded json.dumps form would have escaped the non-ASCII code points.
+    assert json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() == (
+        b'{"id":"caf\\u00e9","msg":"h\\u00fcllo"}'
+    )
+    assert _canonical_bytes(obj) != json.dumps(
+        obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True
+    ).encode()
+
+
+def test_canonical_bytes_known_answer_numbers():
+    """Known-answer vector for JCS / RFC 8785 §3.2.2.3 number serialization.
+
+    ``1e-7`` is the RFC 8785 shortest round-trip form; Python's ``json.dumps``
+    emits ``1e-07`` (zero-padded exponent), so this literal assertion catches a
+    regression to json.dumps for number formatting.
+    """
+    assert _canonical_bytes({"n": 1e-7}) == b'{"n":1e-7}'
+    assert json.dumps(1e-7) == "1e-07"  # the non-conformant form we moved away from
+
+
+def test_canonical_bytes_matches_reference_library():
+    """Cross-check the full record canonicalization against the rfc8785 reference."""
+    record = _minimal_record()
+    record["model"]["note"] = "résumé"  # non-ASCII to exercise the divergence
+    assert _canonical_bytes(record) == rfc8785.dumps(record)
+
+
+def test_jcs_distinguishes_unicode_key_order_from_json_dumps():
+    """JCS sorts keys by UTF-16 code unit; this differs from naive byte sorting.
+
+    Astral-plane code points (here U+1F600, encoded as a surrogate pair in
+    UTF-16) sort AFTER the BMP key ``z`` under UTF-16 code-unit ordering. Python
+    ``json.dumps(sort_keys=True)`` sorts by Unicode code point, placing the
+    astral key (U+1F600) BEFORE ``z`` (U+007A). The two schemes therefore
+    produce different byte sequences for the same object, even though both claim
+    to "sort keys".
+    """
+    obj = {"z": 1, "\U0001f600": 2}
+    jcs = _canonical_bytes(obj)
+    jdump = json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()
+    # JCS (UTF-16 code unit): high surrogate 0xD83D > 0x007A, so "z" comes first.
+    assert jcs == b'{"z":1,"\xf0\x9f\x98\x80":2}'
+    # json.dumps (code point): 0x1F600 > 0x007A as a scalar, so order matches here,
+    # but the emoji is escaped as a surrogate pair, diverging in bytes regardless.
+    assert jcs != jdump
+
+
+def test_round_trip_with_non_ascii_payload():
+    """End-to-end: signing and verifying a record carrying non-ASCII data."""
+    key = generate_key()
+    record = _fresh_record()
+    record["model"]["note"] = "modèle français \U0001f916"
+    signed = sign_record(record, key)
+    verify_record(signed, key_to_jwk(key))  # must not raise