From d7d41ebd62f6d6e134dfc023b5fda182403cc1cd Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Mon, 17 Nov 2025 12:10:55 -0500 Subject: [PATCH] Censor JSON to remove null bytes before inserting into postgres Unfortunately Postgres does not support nulls in JSONB columns, because it doesn't support them in text columns either. This hits us on block 2@5597863, which has a null in its txdata, causing an error on backfill. --- README.org | 8 ++++++++ haskell-src/exec/Chainweb/Lookups.hs | 21 +++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/README.org b/README.org index ed2f61cf..3f02afc0 100644 --- a/README.org +++ b/README.org @@ -330,3 +330,11 @@ manually by undoing migrations and removing ~schema_migrations~ entries. the incoming migrations **before** they upgrade their ~chainweb-data~ versions. This will allow them to detect any potential conflicts and insert new schema migrations to be executed at the right moment, to accommodate the incoming changes. + +* Errata + +chainweb-data is not necessarily able to represent on-chain data in their full fidelity because of representation issues. +Notably: + - null bytes in JSON values cannot be represented in the JSON type in Postgres + (https://www.commandprompt.com/blog/null-characters-workarounds-arent-good-enough/). + chainweb-data replaces them with the escape sequence. diff --git a/haskell-src/exec/Chainweb/Lookups.hs b/haskell-src/exec/Chainweb/Lookups.hs index 1bcd44e5..918685ad 100644 --- a/haskell-src/exec/Chainweb/Lookups.hs +++ b/haskell-src/exec/Chainweb/Lookups.hs @@ -290,6 +290,14 @@ mkCoinbaseEvents height cid blockhash pl = _blockPayloadWithOutputs_coinbase pl bpwoMinerKeys :: BlockPayloadWithOutputs -> [T.Text] bpwoMinerKeys = _minerData_publicKeys . _blockPayloadWithOutputs_minerData +-- Remove null characters and replace them with , because Postgres does not +-- support null characters in JSON and will throw an error on insertion of such +-- values. +censorJSON :: Value -> PgJSONB Value +censorJSON = PgJSONB . transform (over _String censorNulls) + where + censorNulls = T.replace "\0" "" + mkTransaction :: Block -> (CW.Transaction, TransactionOutput) -> Transaction mkTransaction b (tx,txo) = Transaction { _tx_requestKey = DbHash $ hashB64U $ CW._transaction_hash tx @@ -306,16 +314,17 @@ mkTransaction b (tx,txo) = Transaction , _tx_pactId = DbHash . _cont_pactId <$> cnt , _tx_rollback = _cont_rollback <$> cnt , _tx_step = fromIntegral . _cont_step <$> cnt - , _tx_data = (PgJSONB . _cont_data <$> cnt) - <|> (PgJSONB <$> (exc >>= _exec_data)) + , _tx_data = censorJSON <$> + ((_cont_data <$> cnt) + <|> (exc >>= _exec_data)) , _tx_proof = join (_cont_proof <$> cnt) , _tx_gas = fromIntegral $ _toutGas txo , _tx_badResult = badres , _tx_goodResult = goodres , _tx_logs = hashB64U <$> _toutLogs txo - , _tx_metadata = PgJSONB <$> _toutMetaData txo - , _tx_continuation = PgJSONB <$> _toutContinuation txo + , _tx_metadata = censorJSON <$> _toutMetaData txo + , _tx_continuation = censorJSON <$> _toutContinuation txo , _tx_txid = fromIntegral <$> _toutTxId txo , _tx_numEvents = Just $ fromIntegral $ length $ _toutEvents txo } @@ -330,8 +339,8 @@ mkTransaction b (tx,txo) = Transaction ExecPayload _ -> Nothing ContPayload c -> Just c (badres, goodres) = case _toutResult txo of - PactResult (Left v) -> (Just $ PgJSONB v, Nothing) - PactResult (Right v) -> (Nothing, Just $ PgJSONB v) + PactResult (Left v) -> (Just $ censorJSON v, Nothing) + PactResult (Right v) -> (Nothing, Just $ censorJSON v) mkTxEvents :: Int64 -> ChainId -> DbHash BlockHash -> (CW.Transaction,TransactionOutput) -> [Event] mkTxEvents height cid blk (tx,txo) = zipWith (mkEvent cid height blk (Just rk)) (_toutEvents txo) [0..]