fix: Match Cesium's oct encoding of vertex normals

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - Fix `encode` raising `ValueError: Buffer dtype mismatch` with numpy 2 when the `Ellipsoid` axes are numpy float64 scalars. `bounding_sphere` now also accepts float64 positions with the default and `'ritter'` methods.
 - Fix `VertexNormalsExtension` raising `ValueError: Buffer dtype mismatch` for indices that aren't `uint32`, such as the default `int64` from `np.array`. `encode` already accepted any integer dtype.
+- Fix `VertexNormalsExtension` encoding normals pointing exactly along the +x or +y axis (after oct projection) as the opposite direction, and match Cesium's oct encoding, which rounds to the range [0, 255].
 - Package metadata moved to `pyproject.toml`. The `test` extra was removed; development dependencies are now a `dev` dependency group managed with `uv`. See `DEVELOP.md`.
 
 ## [0.5.0] - 2025-06-24

quantized_mesh_encoder/normals.py

@@ -70,8 +70,9 @@ def oct_encode(vec: np.ndarray) -> np.ndarray:
     result[:, 0] = np.where(negative, (1 - np.abs(y)) * sign_not_zero(x), result[:, 0])
     result[:, 1] = np.where(negative, (1 - np.abs(x)) * sign_not_zero(y), result[:, 1])
 
-    # Converts a scalar value in the range [-1.0, 1.0] to a 8-bit 2's complement
-    # number.
-    oct_encoded = np.floor((np.clip(result, -1, 1) * 0.5 + 0.5) * 256).astype(np.uint8)
+    # Converts a scalar value in the range [-1.0, 1.0] to an 8-bit SNORM value
+    # in the range [0, 255], like Cesium's `CesiumMath.toSNorm`. Scaling by 256
+    # would overflow uint8 for a value of exactly 1.0.
+    oct_encoded = np.round((np.clip(result, -1, 1) * 0.5 + 0.5) * 255).astype(np.uint8)
 
     return oct_encoded

test/test_normals.py

@@ -0,0 +1,32 @@
+import numpy as np
+import pytest
+from quantized_mesh_tile.utils import octDecode, octEncode
+
+from quantized_mesh_encoder.normals import oct_encode
+
+AXES = [
+    [1, 0, 0],
+    [-1, 0, 0],
+    [0, 1, 0],
+    [0, -1, 0],
+    [0, 0, 1],
+    [0, 0, -1],
+]
+
+
+@pytest.mark.parametrize("vec", AXES)
+def test_oct_encode_axes(vec):
+    """Unit vectors along each axis survive an encode/decode round trip."""
+    encoded = oct_encode(np.array([vec], dtype=np.float64))
+    decoded = octDecode(*encoded[0].tolist())
+    assert np.allclose(decoded, vec, atol=0.01)
+
+
+def test_oct_encode_matches_reference():
+    """Matches the reference encoder in quantized-mesh-tile (ported from Cesium)."""
+    rng = np.random.default_rng(0)
+    vecs = rng.normal(size=(1000, 3))
+    vecs /= np.linalg.norm(vecs, axis=1)[:, np.newaxis]
+
+    expected = np.array([octEncode(vec.tolist()) for vec in vecs], dtype=np.uint8)
+    assert np.array_equal(oct_encode(vecs), expected)