Use full SHA-256 dedup lookup key

crates/bootstrap_mtc_worker/src/sequence_metadata.rs

@@ -18,12 +18,10 @@ use tlog_entry::LookupKey;
 ///
 /// Wire-format constraints (do not change without migration):
 ///
-/// 1. **Durable Object dedup ring buffer**: 32-byte binary layout
-///    `[16-byte lookup key | 8-byte leaf_index BE | 8-byte timestamp BE]`.
-///    This format matches the one previously used when the type was
-///    `(LeafIndex, UnixTimestampMillis)`; preserving it avoids a one-time deserialize
-///    warning on deploy as the sequencer loads any entries already in DO
-///    storage.
+/// 1. **Durable Object dedup ring buffer**: 48-byte binary layout
+///    `[32-byte lookup key | 8-byte leaf_index BE | 8-byte timestamp BE]`.
+///    The lookup key is the full SHA-256 digest; see
+///    `tlog_entry::LOOKUP_KEY_LEN`.
 /// 2. **DO→Worker RPC**: `bitcode` sequence of the two u64 fields (preserved by
 ///    the tuple-struct layout).
 ///

crates/ct_worker/src/sequence_metadata.rs

@@ -16,8 +16,8 @@ use tlog_entry::LookupKey;
 ///
 /// Wire-format constraints (do not change without migration):
 ///
-/// 1. **Durable Object dedup ring buffer**: 32-byte binary layout
-///    `[16-byte lookup key | 8-byte leaf_index BE | 8-byte timestamp BE]`,
+/// 1. **Durable Object dedup ring buffer**: 48-byte binary layout
+///    `[32-byte lookup key | 8-byte leaf_index BE | 8-byte timestamp BE]`,
 ///    handled via [`serialize_sequence_metadata_entries`] /
 ///    [`deserialize_sequence_metadata_entries`].
 /// 2. **KV long-term dedup cache metadata**: JSON array `[leaf_index, timestamp]`.

crates/generic_log_worker/src/lib.rs

@@ -31,7 +31,7 @@ use serde::de::DeserializeOwned;
 use tlog_checkpoint::UnixTimestampMillis;
 use tlog_core::LeafIndex;
 pub use tlog_entry::LookupKey;
-use tlog_entry::PendingLogEntry;
+use tlog_entry::{LOOKUP_KEY_LEN, PendingLogEntry};
 use tokio::sync::Mutex;
 use util::now_millis;
 use worker::{
@@ -468,15 +468,25 @@ pub trait SequencerMetadata:
 /// [`serialize_sequence_metadata_entries`] / [`deserialize_sequence_metadata_entries`].
 pub type SequenceMetadataEntry = (LookupKey, (LeafIndex, UnixTimestampMillis));
 
+/// Size in bytes of one serialized [`SequenceMetadataEntry`] record in the
+/// fixed binary layout produced by [`serialize_sequence_metadata_entries`]:
+/// `[LOOKUP_KEY_LEN-byte lookup key | 8-byte leaf_index BE | 8-byte timestamp BE]`.
+///
+/// With a 32-byte (full SHA-256) lookup key this is 48 bytes. This value bounds
+/// how many entries a single sequencing round may write into one Durable Object
+/// storage value (see `MAX_POOL_SIZE`), which must stay under the 128 KiB DO
+/// per-value limit.
+pub const CACHE_RECORD_LEN: usize = LOOKUP_KEY_LEN + 16;
+
 /// Serialize a batch of [`SequenceMetadataEntry`] values using the fixed
-/// 32-byte binary format:
+/// [`CACHE_RECORD_LEN`]-byte binary format:
 ///
-/// `[16-byte lookup key | 8-byte leaf_index BE | 8-byte timestamp BE]`
+/// `[LOOKUP_KEY_LEN-byte lookup key | 8-byte leaf_index BE | 8-byte timestamp BE]`
 ///
-/// Used by [`SequencerMetadata`] impls that need to preserve backward
-/// compatibility with existing Durable Object dedup cache storage. Any change
-/// to this format will result in a [`DedupCache::load`] error for deployed
-/// logs.
+/// Used by [`SequencerMetadata`] impls that need a compact fixed-width layout
+/// for the Durable Object dedup cache storage. Any change to this format will
+/// result in a [`DedupCache::load`] error for deployed logs (which is logged
+/// and tolerated: the log continues without the short-term cache).
 ///
 /// # Panics
 ///
@@ -485,7 +495,7 @@ pub type SequenceMetadataEntry = (LookupKey, (LeafIndex, UnixTimestampMillis));
 pub fn serialize_sequence_metadata_entries(entries: &[SequenceMetadataEntry]) -> Vec<u8> {
     use byteorder::{BigEndian, WriteBytesExt};
     use std::io::Write as _;
-    let mut buf = Vec::with_capacity(32 * entries.len());
+    let mut buf = Vec::with_capacity(CACHE_RECORD_LEN * entries.len());
     for (k, (idx, ts)) in entries {
         buf.write_all(k).unwrap();
         buf.write_u64::<BigEndian>(*idx).unwrap();
@@ -494,21 +504,22 @@ pub fn serialize_sequence_metadata_entries(entries: &[SequenceMetadataEntry]) ->
     buf
 }
 
-/// Deserialize a batch of [`SequenceMetadataEntry`] values from the 32-byte
-/// binary format produced by [`serialize_sequence_metadata_entries`].
+/// Deserialize a batch of [`SequenceMetadataEntry`] values from the
+/// [`CACHE_RECORD_LEN`]-byte binary format produced by
+/// [`serialize_sequence_metadata_entries`].
 ///
 /// # Errors
 ///
-/// Returns an error string if `buf.len()` is not a multiple of 32 or if the
-/// underlying byteorder reads fail.
+/// Returns an error string if `buf.len()` is not a multiple of
+/// [`CACHE_RECORD_LEN`] or if the underlying byteorder reads fail.
 pub fn deserialize_sequence_metadata_entries(
     buf: &[u8],
 ) -> std::result::Result<Vec<SequenceMetadataEntry>, String> {
     use byteorder::{BigEndian, ReadBytesExt};
-    if !buf.len().is_multiple_of(32) {
+    if !buf.len().is_multiple_of(CACHE_RECORD_LEN) {
         return Err("invalid buffer length".into());
     }
-    let mut entries = Vec::with_capacity(buf.len() / 32);
+    let mut entries = Vec::with_capacity(buf.len() / CACHE_RECORD_LEN);
     let mut cursor = std::io::Cursor::new(buf);
     while usize::try_from(cursor.position()).unwrap_or(usize::MAX) < buf.len() {
         let mut key = LookupKey::default();
@@ -986,8 +997,8 @@ mod tests {
     fn test_serialize_deserialize_json_roundtrip() {
         // TestMetadata uses the default JSON format; verify entries survive
         // a serialize/deserialize cycle.
-        let key1 = LookupKey::from([1u8; 16]);
-        let key2 = LookupKey::from([2u8; 16]);
+        let key1 = LookupKey::from([1u8; LOOKUP_KEY_LEN]);
+        let key2 = LookupKey::from([2u8; LOOKUP_KEY_LEN]);
         let entries = vec![(key1, TestMetadata(42)), (key2, TestMetadata(99))];
         let serialized = TestMetadata::serialize_cache_entries(&entries);
         let deserialized = TestMetadata::deserialize_cache_entries(&serialized).unwrap();
@@ -1010,7 +1021,7 @@ mod tests {
     #[test]
     fn test_memory_cache_basic_get_put() {
         let cache = MemoryCache::new(10);
-        let key = [1u8; 16];
+        let key = [1u8; LOOKUP_KEY_LEN];
         let metadata = (42u64, 1000u64);
 
         assert!(cache.get_entry(&key).is_none());
@@ -1022,7 +1033,7 @@ mod tests {
     fn test_memory_cache_multiple_entries() {
         let cache = MemoryCache::new(10);
         let entries: Vec<(LookupKey, (u64, u64))> = (0..5u8)
-            .map(|i| ([i; 16], (u64::from(i), u64::from(i) * 100)))
+            .map(|i| ([i; LOOKUP_KEY_LEN], (u64::from(i), u64::from(i) * 100)))
             .collect();
 
         cache.put_entries(&entries);
@@ -1038,23 +1049,23 @@ mod tests {
 
         // Add 5 entries to a cache with max size 3
         for i in 0..5u8 {
-            let key = [i; 16];
+            let key = [i; LOOKUP_KEY_LEN];
             cache.put_entries(&[(key, (u64::from(i), 0))]);
         }
 
         // First 2 entries should be evicted (FIFO order)
-        assert!(cache.get_entry(&[0u8; 16]).is_none());
-        assert!(cache.get_entry(&[1u8; 16]).is_none());
+        assert!(cache.get_entry(&[0u8; LOOKUP_KEY_LEN]).is_none());
+        assert!(cache.get_entry(&[1u8; LOOKUP_KEY_LEN]).is_none());
         // Last 3 should remain
-        assert!(cache.get_entry(&[2u8; 16]).is_some());
-        assert!(cache.get_entry(&[3u8; 16]).is_some());
-        assert!(cache.get_entry(&[4u8; 16]).is_some());
+        assert!(cache.get_entry(&[2u8; LOOKUP_KEY_LEN]).is_some());
+        assert!(cache.get_entry(&[3u8; LOOKUP_KEY_LEN]).is_some());
+        assert!(cache.get_entry(&[4u8; LOOKUP_KEY_LEN]).is_some());
     }
 
     #[test]
     fn test_memory_cache_duplicate_key_not_added() {
         let cache = MemoryCache::new(10);
-        let key = [1u8; 16];
+        let key = [1u8; LOOKUP_KEY_LEN];
 
         cache.put_entries(&[(key, (100, 200))]);
         cache.put_entries(&[(key, (999, 999))]); // Duplicate, should be ignored
@@ -1066,8 +1077,9 @@ mod tests {
     #[test]
     fn test_memory_cache_batch_put() {
         let cache = MemoryCache::new(5);
-        let entries: Vec<(LookupKey, (u64, u64))> =
-            (0..3u8).map(|i| ([i; 16], (u64::from(i), 0))).collect();
+        let entries: Vec<(LookupKey, (u64, u64))> = (0..3u8)
+            .map(|i| ([i; LOOKUP_KEY_LEN], (u64::from(i), 0)))
+            .collect();
 
         cache.put_entries(&entries);
 
@@ -1090,26 +1102,29 @@ mod tests {
 
     // ==================== sequence_metadata binary format Tests ====================
 
-    /// Confirm the shared 32-byte binary cache format is exactly
-    /// `[16-byte key | 8-byte leaf_index BE | 8-byte timestamp BE]`.
-    /// Any change would corrupt deployed workers' DO dedup ring buffer.
+    /// Confirm the shared binary cache format is exactly
+    /// `[LOOKUP_KEY_LEN-byte key | 8-byte leaf_index BE | 8-byte timestamp BE]`
+    /// and that each record is [`CACHE_RECORD_LEN`] bytes wide. Any change would
+    /// corrupt deployed workers' DO dedup ring buffer.
     #[test]
     fn test_sequence_metadata_binary_format_unchanged() {
         let key: LookupKey = [
             0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d,
-            0x0e, 0x0f,
+            0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b,
+            0x1c, 0x1d, 0x1e, 0x1f,
         ];
         let leaf_index: u64 = 0x0102_0304_0506_0708;
         let timestamp: u64 = 0x0a0b_0c0d_0e0f_1011;
 
         let entries = vec![(key, (leaf_index, timestamp))];
         let serialized = serialize_sequence_metadata_entries(&entries);
 
-        let mut expected = Vec::with_capacity(32);
+        let mut expected = Vec::with_capacity(CACHE_RECORD_LEN);
         expected.extend_from_slice(&key);
         expected.extend_from_slice(&leaf_index.to_be_bytes());
         expected.extend_from_slice(&timestamp.to_be_bytes());
 
+        assert_eq!(serialized.len(), CACHE_RECORD_LEN);
         assert_eq!(
             serialized, expected,
             "sequence_metadata binary format has changed — this will corrupt \

crates/generic_log_worker/src/log_ops.rs

@@ -56,10 +56,17 @@ pub const CHECKPOINT_KEY: &str = "checkpoint";
 const STAGING_KEY: &str = "staging";
 
 // Limit on the number of entries per batch. Tune this parameter to avoid
-// running into various size limitations. For instance, unexpectedly large
-// leaves (e.g., with PQ signatures) could cause us to exceed the 128MB Workers
-// memory limit. Storing 4000 10KB certificates is 40MB.
-const MAX_POOL_SIZE: usize = 4000;
+// running into various size limitations:
+//
+//   * Workers isolate memory (128MB): unexpectedly large leaves (e.g., with PQ
+//     signatures) inflate the pool. Storing 2500 10KB certificates is 25MB.
+//   * Durable Object storage per-value limit (128 KiB): each sequencing round
+//     serializes its entries into a single DO storage value in the dedup ring
+//     buffer, at `CACHE_RECORD_LEN` (48) bytes per entry. 2500 * 48 = 120,000
+//     bytes, which stays under the 131,072-byte cap (ceiling is 2730). Because
+//     one round maps to one DO value, this bound doubles as the per-value size
+//     guard, avoiding the need to chunk batches across multiple keys.
+const MAX_POOL_SIZE: usize = 2500;
 
 /// Ephemeral state for pooling entries to the CT log.
 ///

crates/static_ct_api/src/static_ct.rs

@@ -193,14 +193,11 @@ impl PendingLogEntry for StaticCTPendingLogEntry {
         // Add certificate with a 24-bit length prefix
         buffer.write_length_prefixed(&self.certificate, 3).unwrap();
 
-        // Compute the SHA-256 hash of the buffer
-        let hash = Sha256::digest(&buffer);
-
-        // Return the first 16 bytes of the hash as the lookup key.
-        let mut cache_hash = [0u8; 16];
-        cache_hash.copy_from_slice(&hash[..16]);
-
-        cache_hash
+        // Return the full SHA-256 hash of the buffer as the lookup key. Using
+        // the entire digest (rather than a truncation) is a security
+        // requirement; see `tlog_entry::LOOKUP_KEY_LEN` and
+        // <https://github.com/cloudflare/azul/issues/251>.
+        Sha256::digest(&buffer).into()
     }
 }
 

crates/tlog_entry/src/lib.rs

@@ -28,7 +28,15 @@ use tlog_checkpoint::UnixTimestampMillis;
 use tlog_core::{Hash, LeafIndex};
 use tlog_tiles::PathElem;
 
-pub const LOOKUP_KEY_LEN: usize = 16;
+/// Length of a deduplication-cache lookup key, in bytes.
+///
+/// This is the full width of a SHA-256 digest. Using the entire digest (rather
+/// than a truncation) is a security requirement: the lookup key is what binds a
+/// resubmitted certificate to a previously issued SCT, so a collision in the
+/// key lets an attacker obtain an SCT for a certificate that was never
+/// incorporated into the log. See
+/// <https://github.com/cloudflare/azul/issues/251>.
+pub const LOOKUP_KEY_LEN: usize = 32;
 pub type LookupKey = [u8; LOOKUP_KEY_LEN];
 
 /// An opaque `PendingLogEntry` that can be passed around without requiring full
@@ -159,11 +167,8 @@ impl PendingLogEntry for TlogTilesPendingLogEntry {
     }
 
     fn lookup_key(&self) -> LookupKey {
-        let hash = Sha256::digest(&self.data);
-        let mut lookup_key = LookupKey::default();
-        lookup_key.copy_from_slice(&hash[..LOOKUP_KEY_LEN]);
-
-        lookup_key
+        // Use the full SHA-256 digest as the lookup key (see `LOOKUP_KEY_LEN`).
+        Sha256::digest(&self.data).into()
     }
 }