From 7c4854697baa906955b1f54579037e5dcc5e8ba8 Mon Sep 17 00:00:00 2001 From: Rick Date: Wed, 11 Feb 2026 03:50:18 +0000 Subject: [PATCH 1/4] feat: update starknet js to 9.3.0 --- example/reorg.ts | 14 +- package-lock.json | 213 ++++++++++++++++++++++--- package.json | 2 +- src/core/indexer.ts | 18 ++- src/scripts/download-abi.ts | 1 - src/utils/__tests__/blockUtils.test.ts | 1 - 6 files changed, 212 insertions(+), 37 deletions(-) diff --git a/example/reorg.ts b/example/reorg.ts index b646b54..dfba770 100644 --- a/example/reorg.ts +++ b/example/reorg.ts @@ -102,10 +102,16 @@ async function runReorgExample(): Promise { }); // Set up contract with first devnet account - const strkContract = new Contract(universalErc20Abi, CONFIG.CONTRACT_ADDRESS, provider); - strkContract.connect( - new Account(provider, CONFIG.DEVNET_ACCOUNT_1.ADDRESS, CONFIG.DEVNET_ACCOUNT_1.PRIVATE_KEY) - ); + const account = new Account({ + provider, + address: CONFIG.DEVNET_ACCOUNT_1.ADDRESS, + signer: CONFIG.DEVNET_ACCOUNT_1.PRIVATE_KEY, + }); + const strkContract = new Contract({ + abi: universalErc20Abi, + address: CONFIG.CONTRACT_ADDRESS, + providerOrAccount: account, + }); // Perform initial transfers console.log('Performing initial transfers...'); diff --git a/package-lock.json b/package-lock.json index 5074937..35dfff9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "mysql2": "^3.14.1", "patch-package": "^8.0.0", "pg": "^8.11.0", - "starknet": "7.6.2", + "starknet": "9.3.0", "terminal-size": "^4.0.0" }, "bin": { @@ -38,6 +38,12 @@ "typescript": "^5.0.0" } }, + "node_modules/@adraffy/ens-normalize": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.11.1.tgz", + "integrity": "sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==", + "license": "MIT" + }, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", @@ -1377,10 +1383,76 @@ } }, "node_modules/@scure/base": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.2.1.tgz", - "integrity": "sha512-DGmGtC8Tt63J5GfHgfl5CuAXh96VF/LD8K9Hr/Gv0J2lAoRGlPOMpqMpMbCTOoOJMZCk2Xt+DskdDyn6dEFdzQ==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.2.6.tgz", + "integrity": "sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.7.0.tgz", + "integrity": "sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==", "license": "MIT", + "dependencies": { + "@noble/curves": "~1.9.0", + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32/node_modules/@noble/curves": { + "version": "1.9.7", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.7.tgz", + "integrity": "sha512-gbKGcRUYIjA3/zCCNaWDciTMFI0dCkvou3TL8Zmy5Nc7sJ47a0jtOeZoTaMxkuqRo9cRhjOdZJXegxYE5FN/xw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32/node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.6.0.tgz", + "integrity": "sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39/node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, "funding": { "url": "https://paulmillr.com/funding/" } @@ -1425,18 +1497,36 @@ "@sinonjs/commons": "^3.0.0" } }, - "node_modules/@starknet-io/starknet-types-07": { + "node_modules/@starknet-io/get-starknet-wallet-standard": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@starknet-io/get-starknet-wallet-standard/-/get-starknet-wallet-standard-5.0.0.tgz", + "integrity": "sha512-isDNGDlp16W24HE4IuweYXLDRZN0JbsDnazAieeKXE87Mn+jqhsjgTsMxcwWTjX7v906Bjz39FiDjGUddnr36g==", + "license": "MIT", + "dependencies": { + "@starknet-io/types-js": "^0.7.10", + "@wallet-standard/base": "^1.1.0", + "@wallet-standard/features": "^1.1.0", + "ox": "^0.4.4" + } + }, + "node_modules/@starknet-io/starknet-types-010": { "name": "@starknet-io/types-js", - "version": "0.7.10", - "resolved": "https://registry.npmjs.org/@starknet-io/types-js/-/types-js-0.7.10.tgz", - "integrity": "sha512-1VtCqX4AHWJlRRSYGSn+4X1mqolI1Tdq62IwzoU2vUuEE72S1OlEeGhpvd6XsdqXcfHmVzYfj8k1XtKBQqwo9w==", + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@starknet-io/types-js/-/types-js-0.10.0.tgz", + "integrity": "sha512-7ALSydz6pq3YIOpq5a7OkkxqwJciMc9Nlph0OGjhcC3xX0xH30XgizmziLyYVN10oO9+BJk8M9KbJjpzdbtRSw==", "license": "MIT" }, - "node_modules/@starknet-io/starknet-types-08": { + "node_modules/@starknet-io/starknet-types-09": { "name": "@starknet-io/types-js", - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/@starknet-io/types-js/-/types-js-0.8.4.tgz", - "integrity": "sha512-0RZ3TZHcLsUTQaq1JhDSCM8chnzO4/XNsSCozwDET64JK5bjFDIf2ZUkta+tl5Nlbf4usoU7uZiDI/Q57kt2SQ==", + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/@starknet-io/types-js/-/types-js-0.9.2.tgz", + "integrity": "sha512-vWOc0FVSn+RmabozIEWcEny1I73nDGTvOrLYJsR1x7LGA3AZmqt4i/aW69o/3i2NN5CVP8Ok6G1ayRQJKye3Wg==", + "license": "MIT" + }, + "node_modules/@starknet-io/types-js": { + "version": "0.7.10", + "resolved": "https://registry.npmjs.org/@starknet-io/types-js/-/types-js-0.7.10.tgz", + "integrity": "sha512-1VtCqX4AHWJlRRSYGSn+4X1mqolI1Tdq62IwzoU2vUuEE72S1OlEeGhpvd6XsdqXcfHmVzYfj8k1XtKBQqwo9w==", "license": "MIT" }, "node_modules/@types/babel__core": { @@ -1828,6 +1918,27 @@ "dev": true, "license": "ISC" }, + "node_modules/@wallet-standard/base": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@wallet-standard/base/-/base-1.1.0.tgz", + "integrity": "sha512-DJDQhjKmSNVLKWItoKThJS+CsJQjR9AOBOirBVT1F9YpRyC9oYHE+ZnSf8y8bxUphtKqdQMPVQ2mHohYdRvDVQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=16" + } + }, + "node_modules/@wallet-standard/features": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@wallet-standard/features/-/features-1.1.0.tgz", + "integrity": "sha512-hiEivWNztx73s+7iLxsuD1sOJ28xtRix58W7Xnz4XzzA/pF0+aicnWgjOdA10doVDEDZdUuZCIIqG96SFNlDUg==", + "license": "Apache-2.0", + "dependencies": { + "@wallet-standard/base": "^1.1.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/@yarnpkg/lockfile": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", @@ -1849,6 +1960,27 @@ "generate": "dist/generate.js" } }, + "node_modules/abitype": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-1.2.3.tgz", + "integrity": "sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/wevm" + }, + "peerDependencies": { + "typescript": ">=5.0.4", + "zod": "^3.22.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -3237,7 +3369,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", - "dev": true, "license": "MIT" }, "node_modules/execa": { @@ -5534,9 +5665,9 @@ "license": "Apache-2.0" }, "node_modules/lossless-json": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lossless-json/-/lossless-json-4.1.1.tgz", - "integrity": "sha512-HusN80C0ohtT9kOHQH7EuUaqzRQsnekpa+2ot8OzvW0iC08dq/YtM/7uKwwajldQsCrHyC8q9fz3t3L+TmDltA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lossless-json/-/lossless-json-4.3.0.tgz", + "integrity": "sha512-ToxOC+SsduRmdSuoLZLYAr5zy1Qu7l5XhmPWM3zefCZ5IcrzW/h108qbJUKfOlDlhvhjUK84+8PSVX0kxnit0g==", "license": "MIT" }, "node_modules/lru-cache": { @@ -5889,6 +6020,35 @@ "node": ">=0.10.0" } }, + "node_modules/ox": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/ox/-/ox-0.4.4.tgz", + "integrity": "sha512-oJPEeCDs9iNiPs6J0rTx+Y0KGeCGyCAA3zo94yZhm8G5WpOxrwUtn2Ie/Y8IyARSqqY/j9JTKA3Fc1xs1DvFnw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@adraffy/ens-normalize": "^1.10.1", + "@noble/curves": "^1.6.0", + "@noble/hashes": "^1.5.0", + "@scure/bip32": "^1.5.0", + "@scure/bip39": "^1.4.0", + "abitype": "^1.0.6", + "eventemitter3": "5.0.1" + }, + "peerDependencies": { + "typescript": ">=5.4.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -6987,19 +7147,20 @@ } }, "node_modules/starknet": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/starknet/-/starknet-7.6.2.tgz", - "integrity": "sha512-IoXUtzrtG+IPfvnZzIYcbp2lDSCb8VKFyOtgvuJhlfRWUJDxbe27ZAsXJfo9rSFS7kbKny5KdiZgO6RlxcmXvg==", + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/starknet/-/starknet-9.3.0.tgz", + "integrity": "sha512-XtdeESdcy4nXDCFyCqowjGENiBpD5i6ClgvYDqnpvx3SW5PXtKVHRUQVcdkJ6ckz/tQFaH1a1zi7x8ObOoIBdA==", "license": "MIT", "dependencies": { - "@noble/curves": "1.7.0", - "@noble/hashes": "1.6.0", - "@scure/base": "1.2.1", + "@noble/curves": "~1.7.0", + "@noble/hashes": "~1.6.0", + "@scure/base": "~1.2.1", "@scure/starknet": "1.1.0", - "@starknet-io/starknet-types-07": "npm:@starknet-io/types-js@~0.7.10", - "@starknet-io/starknet-types-08": "npm:@starknet-io/types-js@~0.8.4", + "@starknet-io/get-starknet-wallet-standard": "^5.0.0", + "@starknet-io/starknet-types-010": "npm:@starknet-io/types-js@0.10.0", + "@starknet-io/starknet-types-09": "npm:@starknet-io/types-js@~0.9.1", "abi-wan-kanabi": "2.2.4", - "lossless-json": "^4.0.1", + "lossless-json": "^4.2.0", "pako": "^2.0.4", "ts-mixer": "^6.0.3" }, @@ -7427,7 +7588,7 @@ "version": "5.8.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "peer": true, "bin": { diff --git a/package.json b/package.json index c95d826..43301cd 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "mysql2": "^3.14.1", "patch-package": "^8.0.0", "pg": "^8.11.0", - "starknet": "7.6.2", + "starknet": "9.3.0", "terminal-size": "^4.0.0" }, "devDependencies": { diff --git a/src/core/indexer.ts b/src/core/indexer.ts index 49d85c3..0a4e0cd 100644 --- a/src/core/indexer.ts +++ b/src/core/indexer.ts @@ -8,6 +8,7 @@ import { EmittedEvent, WebSocketChannel, Subscription, + createAbiParser, } from 'starknet'; import { @@ -91,7 +92,7 @@ export class StarknetIndexer { } try { - this.provider = new RpcProvider({ nodeUrl: config.rpcNodeUrl, specVersion: '0.8.1' }); + this.provider = new RpcProvider({ nodeUrl: config.rpcNodeUrl }); } catch (error) { this.logger.error('Failed to initialize RPC provider:', error); } @@ -926,6 +927,7 @@ export class StarknetIndexer { block_number: event.block_number, block_hash: event.block_hash || '', transaction_hash: event.transaction_hash, + transaction_index: event.transaction_index ?? 0, from_address: fromAddress, event_index: eventIndex, keys: event.keys, @@ -954,8 +956,15 @@ export class StarknetIndexer { const abiEvents = events.getAbiEvents(abi); const abiStructs = CallData.getAbiStruct(abi); const abiEnums = CallData.getAbiEnum(abi); - - const parsedEvents = events.parseEvents([eventObj], abiEvents, abiStructs, abiEnums); + const parser = createAbiParser(abi); + + const parsedEvents = events.parseEvents( + [eventObj], + abiEvents, + abiStructs, + abiEnums, + parser + ); if (parsedEvents && parsedEvents.length > 0) { // Get the first key of the parsed event (the event name) @@ -966,6 +975,7 @@ export class StarknetIndexer { block_number: eventObj.block_number, block_hash: eventObj.block_hash, transaction_hash: eventObj.transaction_hash, + transaction_index: eventObj.transaction_index, from_address: fromAddress, event_index: eventObj.event_index, keys: eventObj.keys, @@ -975,7 +985,7 @@ export class StarknetIndexer { parsedEvent = parsedEventWithOriginal; // this.logger.debug(`Parsed event values:`, parsedValues); const eventName = eventKey; - this.progressStats.updateEvent(event.block_number, eventName, fromAddress); + this.progressStats.updateEvent(event.block_number ?? 0, eventName, fromAddress); } } catch (error) { this.logger.error(`Error parsing event from contract ${fromAddress}:`, error); diff --git a/src/scripts/download-abi.ts b/src/scripts/download-abi.ts index 9183562..e66d8a1 100644 --- a/src/scripts/download-abi.ts +++ b/src/scripts/download-abi.ts @@ -18,7 +18,6 @@ async function downloadAbi(options: DownloadAbiOptions) { // Initialize RPC provider const provider = new RpcProvider({ nodeUrl: rpcUrl, - specVersion: '0.8.1', }); try { diff --git a/src/utils/__tests__/blockUtils.test.ts b/src/utils/__tests__/blockUtils.test.ts index 61cb43d..865aec0 100644 --- a/src/utils/__tests__/blockUtils.test.ts +++ b/src/utils/__tests__/blockUtils.test.ts @@ -5,7 +5,6 @@ describe('findContractDeploymentBlock', () => { // Use public Starknet Sepolia RPC endpoint for testing const provider = new RpcProvider({ nodeUrl: 'https://starknet-sepolia-rpc.publicnode.com', - specVersion: '0.8.1', }); // Well-known contract on Sepolia testnet From 5a83bac4f2b6ca4309f6b590c0928601f542b42e Mon Sep 17 00:00:00 2001 From: Rick Date: Sun, 1 Mar 2026 12:52:59 +0000 Subject: [PATCH 2/4] feat: halt indexer on block failure instead of retrying Replace the failed-block retry mechanism with a halt-on-failure approach. When a block fails processing, the indexer now halts completely and refuses to process further blocks, preventing data gaps and database inconsistencies. Co-Authored-By: Claude Opus 4.6 --- .../src/core/__tests__/indexer.test.ts | 78 ++++++---- packages/auco-indexer/src/core/indexer.ts | 140 +++++++----------- 2 files changed, 104 insertions(+), 114 deletions(-) diff --git a/packages/auco-indexer/src/core/__tests__/indexer.test.ts b/packages/auco-indexer/src/core/__tests__/indexer.test.ts index f2baedb..355f1c8 100644 --- a/packages/auco-indexer/src/core/__tests__/indexer.test.ts +++ b/packages/auco-indexer/src/core/__tests__/indexer.test.ts @@ -228,11 +228,6 @@ describe('StarknetIndexer', () => { afterEach(async () => { // Clear any timers that might be running - if ((mockIndexer as any).retryTimeout) { - clearTimeout((mockIndexer as any).retryTimeout); - (mockIndexer as any).retryTimeout = undefined; - } - if ((mockIndexer as any).pollTimeout) { clearTimeout((mockIndexer as any).pollTimeout); (mockIndexer as any).pollTimeout = undefined; @@ -804,7 +799,7 @@ describe('StarknetIndexer', () => { }); }); - describe('failed block retry mechanism', () => { + describe('halt on block failure', () => { let mockIndexer: StarknetIndexer; let mockDbHandler: any; let mockProvider: any; @@ -859,16 +854,15 @@ describe('StarknetIndexer', () => { }); afterEach(async () => { - if ((mockIndexer as any).retryTimeout) { - clearTimeout((mockIndexer as any).retryTimeout); - (mockIndexer as any).retryTimeout = undefined; - } (mockIndexer as any).started = false; await mockIndexer.stop(); jest.clearAllMocks(); }); - it('should add failed blocks to retry queue', () => { + it('should halt when a block fails processing', async () => { + // Make insertBlock throw to simulate a block processing failure + mockDbHandler.insertBlock.mockRejectedValueOnce(new Error('DB write failed')); + const blockData = { block_number: 50, block_hash: '0xblock50', @@ -876,38 +870,62 @@ describe('StarknetIndexer', () => { timestamp: Date.now(), }; - (mockIndexer as any).addFailedBlock(blockData, new Error('Processing failed')); + await (mockIndexer as any).processNewHead(blockData); - const failedBlocks = (mockIndexer as any).failedBlocks; - expect(failedBlocks).toContain(50); + expect((mockIndexer as any).halted).toBe(true); + expect((mockIndexer as any).haltReason).toEqual({ + blockNumber: 50, + error: expect.any(Error), + }); }); - it('should not duplicate failed blocks', () => { + it('should not process new blocks after halt', async () => { + // Manually halt the indexer + (mockIndexer as any).halt(42, new Error('test halt')); + const blockData = { - block_number: 50, - block_hash: '0xblock50', - parent_hash: '0xblock49', + block_number: 43, + block_hash: '0xblock43', + parent_hash: '0xblock42', timestamp: Date.now(), }; - (mockIndexer as any).addFailedBlock(blockData, new Error('Processing failed')); - (mockIndexer as any).addFailedBlock(blockData, new Error('Processing failed again')); + await (mockIndexer as any).processNewHead(blockData); - const failedBlocks = (mockIndexer as any).failedBlocks; - expect(failedBlocks.filter((block: number) => block === 50)).toHaveLength(1); + // insertBlock should never be called because the indexer is halted + expect(mockDbHandler.insertBlock).not.toHaveBeenCalled(); }); - it('should retry failed blocks successfully', async () => { - (mockIndexer as any).failedBlocks = [45, 46]; + it('should stop historical block processing when halted', async () => { + // Halt after the first chunk starts + mockDbHandler.checkIsBlockProcessed.mockImplementation(async (blockNumber: number) => { + if (blockNumber >= 1) { + (mockIndexer as any).halt(1, new Error('test halt')); + } + return false; + }); - // Mock processHistoricalBlocks to avoid complex dependencies - (mockIndexer as any).processHistoricalBlocks = jest.fn().mockResolvedValue(undefined); + await (mockIndexer as any).processHistoricalBlocks(1, 200); - await (mockIndexer as any).retryFailedBlocks(); + // Should not have processed the second chunk (blocks 101-200) + // because the indexer halted during or after the first chunk + expect((mockIndexer as any).halted).toBe(true); + }); - expect((mockIndexer as any).processHistoricalBlocks).toHaveBeenCalledWith(45, 45); - expect((mockIndexer as any).processHistoricalBlocks).toHaveBeenCalledWith(46, 46); - expect((mockIndexer as any).failedBlocks).toHaveLength(0); + it('should expose halt state via getHaltState()', () => { + // Initially not halted + const initialState = mockIndexer.getHaltState(); + expect(initialState).toEqual({ halted: false, reason: null }); + + // After halting + (mockIndexer as any).halt(99, new Error('fatal error')); + + const haltedState = mockIndexer.getHaltState(); + expect(haltedState.halted).toBe(true); + expect(haltedState.reason).toEqual({ + blockNumber: 99, + error: expect.any(Error), + }); }); }); diff --git a/packages/auco-indexer/src/core/indexer.ts b/packages/auco-indexer/src/core/indexer.ts index 3b9e293..01ded16 100644 --- a/packages/auco-indexer/src/core/indexer.ts +++ b/packages/auco-indexer/src/core/indexer.ts @@ -61,9 +61,8 @@ export class StarknetIndexer { private readonly dbHandler: BaseDbHandler; private healthCheckInterval?: NodeJS.Timeout; - private failedBlocks: number[] = []; - private retryTimeout?: NodeJS.Timeout; - private readonly RETRY_INTERVAL = 10000; // 10 seconds between retry checks + private halted: boolean = false; + private haltReason: { blockNumber: number; error: any } | null = null; private readonly HEALTH_CHECK_INTERVAL = 3000; // 1 second between health checks private readonly reconnectDelay: number = 1000; private readonly MAX_HISTORICAL_BLOCK_CONCURRENT_REQUESTS: number; @@ -171,6 +170,10 @@ export class StarknetIndexer { return; } + if (this.isHalted()) { + return; + } + await this.withErrorHandling( 'Processing new head', async () => { @@ -500,6 +503,10 @@ export class StarknetIndexer { // Process a new block head private async processNewHead(blockData: any): Promise { + if (this.isHalted()) { + return; + } + if (this.cursor && blockData.block_number <= this.cursor.blockNumber) { if ( blockData.block_number === this.cursor.blockNumber && @@ -510,34 +517,31 @@ export class StarknetIndexer { } } - try { - await this.withTransaction( - 'Processing block', - async () => { - await this.insertBlock(blockData); - await this.updateCursor(blockData.block_number, blockData.block_hash); - this.logger.info(`Successfully processed block #${blockData.block_number}`); - - if (this.provider) { - await this.processBlockEvents( - blockData.block_number, - blockData.block_number, - blockData.block_hash - ); - } + const result = await this.withTransaction( + 'Processing block', + async () => { + await this.insertBlock(blockData); + await this.updateCursor(blockData.block_number, blockData.block_hash); + this.logger.info(`Successfully processed block #${blockData.block_number}`); + + if (this.provider) { + await this.processBlockEvents( + blockData.block_number, + blockData.block_number, + blockData.block_hash + ); + } - if (this.failedBlocks.length > 0 && this.failedBlocks.includes(blockData.block_number)) { - this.failedBlocks = this.failedBlocks.filter( - (block) => block !== blockData.block_number - ); - } - }, - { blockNumber: blockData.block_number } + return true; // signal success + }, + { blockNumber: blockData.block_number } + ); + + if (result === undefined) { + this.halt( + blockData.block_number, + new Error(`Transaction failed for block #${blockData.block_number}`) ); - // No need to call updateEvent for blocks unless you want to track them as events - } catch (error) { - this.logger.error(`Failed to process block #${blockData.block_number}:`, error); - this.addFailedBlock(blockData, error); } } @@ -613,10 +617,6 @@ export class StarknetIndexer { clearInterval(this.healthCheckInterval); } - if (this.retryTimeout) { - clearTimeout(this.retryTimeout); - this.retryTimeout = undefined; - } this.stopProgressUiLoop(); // Unsubscribe from new heads subscription to stop event processing @@ -701,6 +701,7 @@ export class StarknetIndexer { } private async processBlockQueue(): Promise { + if (this.isHalted()) return; if (this.blockQueue.length === 0) return; const TAG = 'processBlockQueue'; const blocksToProcess = [...this.blockQueue]; @@ -823,6 +824,11 @@ export class StarknetIndexer { this.progressStats.initSyncStats(fromBlock, toBlock); for (let blockNumber = fromBlock; blockNumber <= toBlock; blockNumber += chunkSize) { + if (this.isHalted()) { + this.logger.warn(`[${TAG}] Halted — aborting historical block sync`); + return; + } + const chunkEndBlock = Math.min(blockNumber + chunkSize - 1, toBlock); const chunkLabel = `blocks_${blockNumber}_to_${chunkEndBlock}`; @@ -1082,62 +1088,28 @@ export class StarknetIndexer { return await this.dbHandler.checkIsBlockProcessed(blockNumber); } - private addFailedBlock(blockData: QueuedBlock, _error: any): void { - if (!this.failedBlocks.includes(blockData.block_number)) { - this.failedBlocks.push(blockData.block_number); - this.logger.warn(`Added block #${blockData.block_number} to failed blocks queue`); - } - - // Start retry process if it's not already running - if (!this.retryTimeout) { - this.startRetryProcess(); - } - } - - private async retryFailedBlocks(): Promise { - if (this.failedBlocks.length === 0) return; - - const latestBlock = await this.provider!.getBlock('latest'); - const blocksToRetry = this.failedBlocks.filter( - (blockNumber) => blockNumber < latestBlock.block_number + private halt(blockNumber: number, error: any): void { + this.halted = true; + this.haltReason = { blockNumber, error }; + this.logger.error( + `[HALTED] Indexer halted due to failure at block #${blockNumber}. ` + + `Error: ${error instanceof Error ? error.message : String(error)}. ` + + `The indexer will not process any further blocks. Resolve the issue and restart.` ); + } - if (blocksToRetry.length === 0) return; - - this.logger.info(`Attempting to retry ${blocksToRetry.length} failed blocks`); - - for (const blockNumber of blocksToRetry) { - try { - await this.processHistoricalBlocks(blockNumber, blockNumber); - // Remove from failed blocks if successful - this.failedBlocks = this.failedBlocks.filter((b) => b !== blockNumber); - } catch (error) { - this.logger.error(`Retry failed for block #${blockNumber}:`, error); - } + private isHalted(): boolean { + if (this.halted) { + this.logger.warn( + `[HALTED] Indexer is halted (block #${this.haltReason?.blockNumber}). Ignoring request to process.` + ); + return true; } + return false; } - private startRetryProcess(): void { - const retryProcess = async () => { - if (!this.started) return; - - try { - await this.retryFailedBlocks(); - } catch (error) { - this.logger.error('Error in retry process:', error); - } - - // Only schedule next retry if there are failed blocks - if (this.failedBlocks.length > 0) { - this.retryTimeout = setTimeout(retryProcess, this.RETRY_INTERVAL); - } else { - this.logger.debug('No failed blocks to retry, stopping retry process'); - clearTimeout(this.retryTimeout); - this.retryTimeout = undefined; - } - }; - - retryProcess(); + public getHaltState(): { halted: boolean; reason: { blockNumber: number; error: any } | null } { + return { halted: this.halted, reason: this.haltReason }; } private startProgressUiLoop() { From dd7d0774c892dffffdadaac4d7894e3b6352726f Mon Sep 17 00:00:00 2001 From: Rick Date: Wed, 4 Mar 2026 09:47:41 +0000 Subject: [PATCH 3/4] feat: add retry mechanism and partial progress to halt behavior - Add automatic retry of failed blocks on a 10s timer - Resume indexer automatically when all retries succeed - Refactor processHistoricalBlocks to per-block transactions for partial progress (blocks before failure are committed) - Update cursor per-block during historical sync for correct restart - Add tests for retry queue, resume, dedup, and partial progress Co-Authored-By: Claude Opus 4.6 --- .../src/core/__tests__/indexer.test.ts | 108 +++++++++++++- packages/auco-indexer/src/core/indexer.ts | 135 ++++++++++++++---- 2 files changed, 213 insertions(+), 30 deletions(-) diff --git a/packages/auco-indexer/src/core/__tests__/indexer.test.ts b/packages/auco-indexer/src/core/__tests__/indexer.test.ts index 355f1c8..8496e05 100644 --- a/packages/auco-indexer/src/core/__tests__/indexer.test.ts +++ b/packages/auco-indexer/src/core/__tests__/indexer.test.ts @@ -799,7 +799,7 @@ describe('StarknetIndexer', () => { }); }); - describe('halt on block failure', () => { + describe('halt and retry on block failure', () => { let mockIndexer: StarknetIndexer; let mockDbHandler: any; let mockProvider: any; @@ -854,6 +854,10 @@ describe('StarknetIndexer', () => { }); afterEach(async () => { + if ((mockIndexer as any).retryTimeout) { + clearTimeout((mockIndexer as any).retryTimeout); + (mockIndexer as any).retryTimeout = undefined; + } (mockIndexer as any).started = false; await mockIndexer.stop(); jest.clearAllMocks(); @@ -927,6 +931,108 @@ describe('StarknetIndexer', () => { error: expect.any(Error), }); }); + + it('should add failed block to retry queue when halting', () => { + (mockIndexer as any).halt(42, new Error('test halt')); + + expect((mockIndexer as any).failedBlocks).toContain(42); + expect((mockIndexer as any).retryTimeout).toBeDefined(); + }); + + it('should not duplicate failed blocks in retry queue', () => { + (mockIndexer as any).halt(42, new Error('test halt 1')); + (mockIndexer as any).halt(42, new Error('test halt 2')); + + const failedBlocks = (mockIndexer as any).failedBlocks; + expect(failedBlocks.filter((b: number) => b === 42)).toHaveLength(1); + }); + + it('should resume after successful retry', async () => { + // Halt the indexer + (mockIndexer as any).halt(50, new Error('temporary failure')); + expect((mockIndexer as any).halted).toBe(true); + + // Mock getBlock to return a valid block for the retry + mockProvider.getBlock.mockResolvedValue({ + block_number: 50, + block_hash: '0xblock50', + parent_hash: '0xblock49', + timestamp: Date.now(), + }); + + // Mock withExponentialBackoff to just run the function + (mockIndexer as any).withExponentialBackoff = jest + .fn() + .mockImplementation(async (_op: string, fn: () => Promise) => { + return await fn(); + }); + + // Clear halt state to allow retry to run (simulating what startRetryProcess does) + (mockIndexer as any).halted = false; + (mockIndexer as any).haltReason = null; + + await (mockIndexer as any).retryFailedBlocks(); + + expect((mockIndexer as any).halted).toBe(false); + expect((mockIndexer as any).haltReason).toBeNull(); + expect((mockIndexer as any).failedBlocks).toHaveLength(0); + }); + + it('should commit partial progress — blocks before failure are committed', async () => { + // Setup: blocks 1-5, block 3 fails + const blocks = [1, 2, 3, 4, 5]; + + mockDbHandler.checkIsBlockProcessed.mockResolvedValue(false); + + mockProvider.getBlock.mockImplementation(async (blockNumber: number) => ({ + block_number: blockNumber, + block_hash: `0xblock${blockNumber}`, + parent_hash: `0xblock${blockNumber - 1}`, + timestamp: Date.now(), + })); + + // Make the transaction fail for block 3 + let insertCallCount = 0; + mockDbHandler.withTransaction.mockImplementation(async (fn: () => Promise) => { + insertCallCount++; + if (insertCallCount === 3) { + // Block 3's transaction fails + throw new Error('DB write failed for block 3'); + } + return await fn(); + }); + + // Mock withExponentialBackoff to just run the function + (mockIndexer as any).withExponentialBackoff = jest + .fn() + .mockImplementation(async (_op: string, fn: () => Promise) => { + return await fn(); + }); + + await (mockIndexer as any).processHistoricalBlocks(1, 5); + + // Blocks 1 and 2 should have been inserted (their transactions succeeded) + expect(mockDbHandler.insertBlock).toHaveBeenCalledTimes(2); + expect(mockDbHandler.insertBlock).toHaveBeenCalledWith( + expect.objectContaining({ block_number: 1 }) + ); + expect(mockDbHandler.insertBlock).toHaveBeenCalledWith( + expect.objectContaining({ block_number: 2 }) + ); + + // Cursor should have been updated for blocks 1 and 2 + expect(mockDbHandler.updateCursor).toHaveBeenCalledWith(1, '0xblock1', undefined); + expect(mockDbHandler.updateCursor).toHaveBeenCalledWith(2, '0xblock2', undefined); + + // Block 3 should NOT have been inserted (its transaction failed) + expect(mockDbHandler.insertBlock).not.toHaveBeenCalledWith( + expect.objectContaining({ block_number: 3 }) + ); + + // Indexer should be halted at block 3 + expect((mockIndexer as any).halted).toBe(true); + expect((mockIndexer as any).haltReason?.blockNumber).toBe(3); + }); }); describe('exponential backoff retry', () => { diff --git a/packages/auco-indexer/src/core/indexer.ts b/packages/auco-indexer/src/core/indexer.ts index 01ded16..f5195ed 100644 --- a/packages/auco-indexer/src/core/indexer.ts +++ b/packages/auco-indexer/src/core/indexer.ts @@ -13,7 +13,6 @@ import { import { findContractDeploymentBlock, - groupConsecutiveBlocks, parallelMap, } from '../utils/blockUtils'; import { @@ -63,6 +62,9 @@ export class StarknetIndexer { private halted: boolean = false; private haltReason: { blockNumber: number; error: any } | null = null; + private failedBlocks: number[] = []; + private retryTimeout?: NodeJS.Timeout; + private readonly RETRY_INTERVAL = 10000; private readonly HEALTH_CHECK_INTERVAL = 3000; // 1 second between health checks private readonly reconnectDelay: number = 1000; private readonly MAX_HISTORICAL_BLOCK_CONCURRENT_REQUESTS: number; @@ -613,6 +615,11 @@ export class StarknetIndexer { clearTimeout(this.pollTimeout); } + if (this.retryTimeout) { + clearTimeout(this.retryTimeout); + this.retryTimeout = undefined; + } + if (this.healthCheckInterval) { clearInterval(this.healthCheckInterval); } @@ -878,37 +885,43 @@ export class StarknetIndexer { const fetchDuration = Date.now() - time; this.logger.debug(`[${TAG}] Fetched ${blocks.length} blocks in ${fetchDuration}ms`); - // Process all blocks and their events in a single transaction - await this.withTransaction( - `Processing blocks ${blockNumber} to ${chunkEndBlock} and their events`, - async () => { - const insertStart = Date.now(); + if (blocks.length === 0) { + this.logger.info( + `[${TAG}] Skipping blocks ${blockNumber} to ${chunkEndBlock} - already processed` + ); + continue; + } - // Batch insert all blocks - if (blocks.length > 0) { - await this.dbHandler.batchInsertBlocks(blocks); + // Sort blocks by number to ensure correct ordering for partial progress + blocks.sort((a, b) => a.block_number - b.block_number); - this.logger.debug( - `[${TAG}] Inserted ${blocks.length} blocks in ${Date.now() - insertStart}ms` - ); - const eventsStart = Date.now(); + // Process each block individually in its own transaction for partial progress + for (const block of blocks) { + if (this.isHalted()) { + return; + } - if (blocks.length < chunkSize) { - const blockRanges = groupConsecutiveBlocks(blocks.map((block) => block.block_number)); - for (const range of blockRanges) { - await this.processBlockEvents(range.from, range.to); - } - } else { - await this.processBlockEvents(blockNumber, chunkEndBlock); - } - this.logger.debug(`[${TAG}] Events processed in ${Date.now() - eventsStart}ms`); - } else { - this.logger.info( - `[${TAG}] Skipping blocks ${blockNumber} to ${chunkEndBlock} - already processed` - ); - } + const result = await this.withTransaction( + `Processing block #${block.block_number}`, + async () => { + await this.insertBlock(block); + await this.processBlockEvents(block.block_number, block.block_number); + await this.updateCursor(block.block_number, block.block_hash); + return true; + }, + { blockNumber: block.block_number } + ); + + if (result === undefined) { + this.halt( + block.block_number, + new Error(`Transaction failed for block #${block.block_number}`) + ); + return; } - ); + + this.logger.debug(`[${TAG}] Committed block #${block.block_number}`); + } } } @@ -1091,11 +1104,75 @@ export class StarknetIndexer { private halt(blockNumber: number, error: any): void { this.halted = true; this.haltReason = { blockNumber, error }; + + if (!this.failedBlocks.includes(blockNumber)) { + this.failedBlocks.push(blockNumber); + } + this.logger.error( `[HALTED] Indexer halted due to failure at block #${blockNumber}. ` + `Error: ${error instanceof Error ? error.message : String(error)}. ` + - `The indexer will not process any further blocks. Resolve the issue and restart.` + `Will retry automatically every ${this.RETRY_INTERVAL / 1000}s.` ); + + if (!this.retryTimeout) { + this.startRetryProcess(); + } + } + + private resume(): void { + this.halted = false; + this.haltReason = null; + this.logger.info('[RESUMED] Indexer resumed after successful retry of all failed blocks.'); + } + + private async retryFailedBlocks(): Promise { + const blocksToRetry = [...this.failedBlocks]; + this.logger.info(`[RETRY] Attempting to retry ${blocksToRetry.length} failed block(s): ${blocksToRetry.join(', ')}`); + + for (const blockNumber of blocksToRetry) { + try { + await this.processHistoricalBlocks(blockNumber, blockNumber); + + if (!this.halted) { + // Block succeeded — remove from failed list + this.failedBlocks = this.failedBlocks.filter((b) => b !== blockNumber); + this.logger.info(`[RETRY] Block #${blockNumber} succeeded on retry.`); + } else { + // processHistoricalBlocks re-halted on this block, stop retrying further + this.logger.warn(`[RETRY] Block #${blockNumber} still failing.`); + return; + } + } catch (error) { + this.logger.error(`[RETRY] Unexpected error retrying block #${blockNumber}:`, error); + return; + } + } + + if (this.failedBlocks.length === 0) { + this.resume(); + } + } + + private startRetryProcess(): void { + this.retryTimeout = setTimeout(async () => { + this.retryTimeout = undefined; + + if (this.failedBlocks.length === 0) { + return; + } + + // Temporarily clear halt so processHistoricalBlocks can run + this.halted = false; + this.haltReason = null; + + await this.retryFailedBlocks(); + + // If still have failed blocks, schedule next retry + if (this.failedBlocks.length > 0) { + this.startRetryProcess(); + } + }, this.RETRY_INTERVAL); } private isHalted(): boolean { From 309e39cfabf7d7bc59892763b9c875ee2e46fb3e Mon Sep 17 00:00:00 2001 From: Rick Date: Wed, 4 Mar 2026 10:04:17 +0000 Subject: [PATCH 4/4] feat: make retry interval configurable via IndexerConfig Add `retryInterval` option to IndexerConfig (default: 10000ms) instead of hardcoding the retry interval. Co-Authored-By: Claude Opus 4.6 --- packages/auco-indexer/src/core/indexer.ts | 4 +++- packages/auco-indexer/src/types/indexer.ts | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/auco-indexer/src/core/indexer.ts b/packages/auco-indexer/src/core/indexer.ts index f5195ed..db9a08a 100644 --- a/packages/auco-indexer/src/core/indexer.ts +++ b/packages/auco-indexer/src/core/indexer.ts @@ -64,7 +64,7 @@ export class StarknetIndexer { private haltReason: { blockNumber: number; error: any } | null = null; private failedBlocks: number[] = []; private retryTimeout?: NodeJS.Timeout; - private readonly RETRY_INTERVAL = 10000; + private readonly RETRY_INTERVAL: number; private readonly HEALTH_CHECK_INTERVAL = 3000; // 1 second between health checks private readonly reconnectDelay: number = 1000; private readonly MAX_HISTORICAL_BLOCK_CONCURRENT_REQUESTS: number; @@ -102,6 +102,8 @@ export class StarknetIndexer { this.MAX_HISTORICAL_BLOCK_CONCURRENT_REQUESTS = config.maxHistoricalBlockConcurrentRequests ?? 5; + this.RETRY_INTERVAL = config.retryInterval ?? 10000; + // Warn if dev mode reset is enabled if (config.devMode?.resetOnStart) { this.logger.warn( diff --git a/packages/auco-indexer/src/types/indexer.ts b/packages/auco-indexer/src/types/indexer.ts index 2a44f99..9911ad5 100644 --- a/packages/auco-indexer/src/types/indexer.ts +++ b/packages/auco-indexer/src/types/indexer.ts @@ -108,6 +108,13 @@ export interface IndexerConfig { */ maxHistoricalBlockConcurrentRequests?: number; + /** + * Interval in milliseconds between retry attempts for failed blocks (default: 10000) + * When a block fails processing, the indexer halts and retries the failed block(s) + * on this interval until they succeed. + */ + retryInterval?: number; + /** Enable UI progress bar for tracking indexer progress */ enableUiProgress?: boolean;