diff --git a/apps/ensindexer/package.json b/apps/ensindexer/package.json index 1db3a2711..0f22cb35c 100644 --- a/apps/ensindexer/package.json +++ b/apps/ensindexer/package.json @@ -29,6 +29,7 @@ "@ensnode/ensnode-sdk": "workspace:*", "@ensnode/ensrainbow-sdk": "workspace:*", "@ensnode/ponder-metadata": "workspace:*", + "@ensnode/ponder-sdk": "workspace:*", "caip": "catalog:", "date-fns": "catalog:", "deepmerge-ts": "^7.1.5", diff --git a/apps/ensindexer/src/lib/indexing-status-builder/backfill-scope.ts b/apps/ensindexer/src/lib/indexing-status-builder/backfill-scope.ts new file mode 100644 index 000000000..29cf4ff61 --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/backfill-scope.ts @@ -0,0 +1,24 @@ +import type { BlockRef } from "@ensnode/ponder-sdk"; + +/** + * Backfill scope for a chain. + * + * Defines the range of blocks to be indexed during the backfill phase. + * The backfill phase progresses from `startBlock` toward `endBlock`, and ends + * when the indexed block reaches `endBlock`. + */ +export interface BackfillScope { + /** + * The starting block of the backfill range (inclusive). + * This is derived from the chain's indexing configuration. + */ + startBlock: BlockRef; + + /** + * The ending block of the backfill range (inclusive). + * This is the "fixed target" that backfill progresses toward. + * + * Guaranteed to be greater than `startBlock`. + */ + endBlock: BlockRef; +} diff --git a/apps/ensindexer/src/lib/indexing-status-builder/block-refs.mock.ts b/apps/ensindexer/src/lib/indexing-status-builder/block-refs.mock.ts new file mode 100644 index 000000000..7fe06113e --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/block-refs.mock.ts @@ -0,0 +1,21 @@ +import type { BlockRef } from "@ensnode/ponder-sdk"; + +export const earliestBlockRef = { + timestamp: 1667260799, + number: 999, +} as const satisfies BlockRef; + +export const earlierBlockRef = { + timestamp: 1672531199, + number: 1024, +} as const satisfies BlockRef; + +export const laterBlockRef = { + timestamp: 1672531200, + number: 1025, +} as const satisfies BlockRef; + +export const latestBlockRef = { + timestamp: 1677104542, + number: 1222, +} as const satisfies BlockRef; diff --git a/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-metadata.mock.ts b/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-metadata.mock.ts new file mode 100644 index 000000000..7317abf7a --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-metadata.mock.ts @@ -0,0 +1,87 @@ +import { type ChainIndexingConfig, ChainIndexingConfigTypeIds } from "@ensnode/ensnode-sdk"; +import type { + BlockRef, + ChainIndexingMetrics, + ChainIndexingMetricsCompleted, + ChainIndexingMetricsHistorical, + ChainIndexingMetricsRealtime, + ChainIndexingState, +} from "@ensnode/ponder-sdk"; +import { ChainIndexingStates } from "@ensnode/ponder-sdk"; + +import type { BackfillScope } from "./backfill-scope"; +import type { ChainIndexingMetadata } from "./chain-indexing-metadata"; + +/** + * Build complete indexing metadata for a chain, used for testing the indexing + * status snapshot builders. + * + * @param config - The chain indexing configuration, including backfill scope. + * @param checkpointBlock - The block ref representing the current checkpoint of + * indexing. + * @param state - The current state of indexing (Historical, Realtime, Completed). + * @param latestSyncedBlock - The latest block ref that has been synced (used for + * Realtime state). + * @param backfillEndBlock - The block ref representing the end of the backfill + * scope (used for Historical state). + * + * @returns Complete indexing metadata for the chain. + */ +export function buildChainIndexingMetadataMock({ + config, + checkpointBlock, + state, + latestSyncedBlock, + backfillEndBlock, +}: { + config: ChainIndexingConfig; + checkpointBlock: BlockRef; + state: ChainIndexingState; + latestSyncedBlock: BlockRef; + backfillEndBlock: BlockRef; +}): ChainIndexingMetadata { + // assuming historical total blocks as + // `latestBlockRef.number - config.startBlock.number + 1` for the sake of + // this mock, since we don't have real metrics to determine + // the actual end block for historical indexing. + const historicalTotalBlocks = backfillEndBlock.number - config.startBlock.number + 1; + + const backfillScope = { + startBlock: config.startBlock, + endBlock: + config.configType === ChainIndexingConfigTypeIds.Definite + ? config.endBlock + : backfillEndBlock, + } satisfies BackfillScope; + + let indexingMetrics: ChainIndexingMetrics; + + switch (state) { + case ChainIndexingStates.Historical: + indexingMetrics = { + state: ChainIndexingStates.Historical, + latestSyncedBlock, + historicalTotalBlocks, + } satisfies ChainIndexingMetricsHistorical; + break; + case ChainIndexingStates.Realtime: + indexingMetrics = { + state: ChainIndexingStates.Realtime, + latestSyncedBlock: latestSyncedBlock, + } satisfies ChainIndexingMetricsRealtime; + break; + case ChainIndexingStates.Completed: + indexingMetrics = { + state: ChainIndexingStates.Completed, + finalIndexedBlock: checkpointBlock, + } satisfies ChainIndexingMetricsCompleted; + break; + } + + return { + backfillScope, + indexingConfig: config, + indexingMetrics, + indexingStatus: { checkpointBlock }, + }; +} diff --git a/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-metadata.ts b/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-metadata.ts new file mode 100644 index 000000000..e37c1459b --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-metadata.ts @@ -0,0 +1,43 @@ +import type { ChainIndexingConfig } from "@ensnode/ensnode-sdk"; +import type { ChainIndexingMetrics, ChainIndexingStatus } from "@ensnode/ponder-sdk"; + +import type { BackfillScope } from "./backfill-scope"; + +/** + * Complete indexing metadata for a single chain. + * + * Bundles all data sources describing the chain indexing that are needed to + * build a ChainIndexingStatusSnapshot: + * - The configured backfill scope (what blocks we need to index) + * - Ponder's indexing metrics (current progress/performance) + * - Ponder's indexing status (checkpoint state) + */ +export interface ChainIndexingMetadata { + /** + * Backfill scope for the chain + * + * Defines the range of blocks to be indexed during the backfill phase. + */ + backfillScope: BackfillScope; + + /** + * Indexing config for the chain + * + * Defines the range of blocks to be indexed for the chain. + */ + indexingConfig: ChainIndexingConfig; + + /** + * Indexing metrics for the chain + * + * References current indexing metrics from Ponder Metrics. + */ + indexingMetrics: ChainIndexingMetrics; + + /** + * Indexing status for this chain + * + * References the checkpoint block from Ponder Status. + */ + indexingStatus: ChainIndexingStatus; +} diff --git a/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-status-snapshot.test.ts b/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-status-snapshot.test.ts new file mode 100644 index 000000000..d520bf270 --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-status-snapshot.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from "vitest"; + +import { + ChainIndexingConfigTypeIds, + ChainIndexingStatusIds, + type ChainIndexingStatusSnapshotBackfill, + type ChainIndexingStatusSnapshotCompleted, + type ChainIndexingStatusSnapshotFollowing, + type ChainIndexingStatusSnapshotQueued, + createIndexingConfig, +} from "@ensnode/ensnode-sdk"; +import { ChainIndexingStates } from "@ensnode/ponder-sdk"; + +import { + earlierBlockRef, + earliestBlockRef, + laterBlockRef, + latestBlockRef, +} from "./block-refs.mock"; +import type { ChainIndexingMetadata } from "./chain-indexing-metadata"; +import { buildChainIndexingMetadataMock } from "./chain-indexing-metadata.mock"; +import { buildChainStatusSnapshots } from "./chain-indexing-status-snapshot"; + +describe("ChainIndexingStatusSnapshot", () => { + describe("buildChainStatusSnapshots", () => { + it("returns Queued status when checkpointBlock equals startBlock", () => { + // arrange + const metadata = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, latestBlockRef), + checkpointBlock: earliestBlockRef, + state: ChainIndexingStates.Historical, + latestSyncedBlock: earlierBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const chainsMetadata = new Map([[1, metadata]]); + + // act + const result = buildChainStatusSnapshots(chainsMetadata); + + // assert + expect(result.get(1)).toStrictEqual({ + chainStatus: ChainIndexingStatusIds.Queued, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + } satisfies ChainIndexingStatusSnapshotQueued); + }); + + it("returns Completed status when indexingMetrics.state is Completed", () => { + // arrange + const metadata = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, latestBlockRef), + checkpointBlock: latestBlockRef, + state: ChainIndexingStates.Completed, + latestSyncedBlock: latestBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const chainsMetadata = new Map([[1, metadata]]); + + // act + const result = buildChainStatusSnapshots(chainsMetadata); + + // assert + expect(result.get(1)).toStrictEqual({ + chainStatus: ChainIndexingStatusIds.Completed, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotCompleted); + }); + + it("returns Following status when indexingMetrics.state is Realtime", () => { + // arrange + const metadata = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, null), + checkpointBlock: laterBlockRef, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: latestBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const chainsMetadata = new Map([[1, metadata]]); + + // act + const result = buildChainStatusSnapshots(chainsMetadata); + + // assert + expect(result.get(1)).toStrictEqual({ + chainStatus: ChainIndexingStatusIds.Following, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: earliestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + latestKnownBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotFollowing); + }); + + it("returns Backfill status for Historical state with definite config", () => { + // arrange + const metadata = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, latestBlockRef), + checkpointBlock: laterBlockRef, + state: ChainIndexingStates.Historical, + latestSyncedBlock: latestBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const chainsMetadata = new Map([[1, metadata]]); + + // act + const result = buildChainStatusSnapshots(chainsMetadata); + + // assert + expect(result.get(1)).toStrictEqual({ + chainStatus: ChainIndexingStatusIds.Backfill, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + backfillEndBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotBackfill); + }); + + it("returns Backfill status for Historical state with indefinite config", () => { + // arrange + const metadata = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, null), + checkpointBlock: earlierBlockRef, + state: ChainIndexingStates.Historical, + latestSyncedBlock: laterBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const chainsMetadata = new Map([[1, metadata]]); + + // act + const result = buildChainStatusSnapshots(chainsMetadata); + + // assert + expect(result.get(1)).toStrictEqual({ + chainStatus: ChainIndexingStatusIds.Backfill, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: earliestBlockRef, + }, + latestIndexedBlock: earlierBlockRef, + backfillEndBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotBackfill); + }); + + it("returns empty map when no chains provided", () => { + // arrange + const chainsMetadata = new Map(); + + // act + const result = buildChainStatusSnapshots(chainsMetadata); + + // assert + expect(result.size).toBe(0); + }); + }); +}); diff --git a/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-status-snapshot.ts b/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-status-snapshot.ts new file mode 100644 index 000000000..7704a90b3 --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/chain-indexing-status-snapshot.ts @@ -0,0 +1,86 @@ +import { + type ChainIndexingConfigDefinite, + type ChainIndexingConfigIndefinite, + ChainIndexingStatusIds, + type ChainIndexingStatusSnapshot, + type ChainIndexingStatusSnapshotBackfill, + type ChainIndexingStatusSnapshotCompleted, + type ChainIndexingStatusSnapshotFollowing, + type ChainIndexingStatusSnapshotQueued, + type Unvalidated, + validateChainIndexingStatusSnapshot, +} from "@ensnode/ensnode-sdk"; +import { type ChainId, ChainIndexingStates, isBlockRefEqualTo } from "@ensnode/ponder-sdk"; + +import type { ChainIndexingMetadata } from "./chain-indexing-metadata"; + +/** + * Build Chain Indexing Status Snapshot from metadata. + * + * @param metadata - Complete indexing metadata including backfill scope, + * Ponder config, metrics, and status needed to determine the chain's state. + * + * @returns The chain indexing status snapshot. + */ +function buildChainIndexingStatusSnapshot( + metadata: ChainIndexingMetadata, +): ChainIndexingStatusSnapshot { + const { backfillScope, indexingConfig, indexingMetrics, indexingStatus } = metadata; + const { checkpointBlock } = indexingStatus; + + // In omnichain ordering, if the startBlock is the same as the + // status block, the chain has not started yet. + if (isBlockRefEqualTo(backfillScope.startBlock, checkpointBlock)) { + return validateChainIndexingStatusSnapshot({ + chainStatus: ChainIndexingStatusIds.Queued, + config: indexingConfig, + } satisfies Unvalidated); + } + + if (indexingMetrics.state === ChainIndexingStates.Completed) { + return validateChainIndexingStatusSnapshot({ + chainStatus: ChainIndexingStatusIds.Completed, + latestIndexedBlock: checkpointBlock, + config: indexingConfig as Unvalidated, + } satisfies Unvalidated); + } + + if (indexingMetrics.state === ChainIndexingStates.Realtime) { + return validateChainIndexingStatusSnapshot({ + chainStatus: ChainIndexingStatusIds.Following, + latestIndexedBlock: checkpointBlock, + latestKnownBlock: indexingMetrics.latestSyncedBlock, + config: indexingConfig as Unvalidated, + } satisfies Unvalidated); + } + + return validateChainIndexingStatusSnapshot({ + chainStatus: ChainIndexingStatusIds.Backfill, + latestIndexedBlock: checkpointBlock, + backfillEndBlock: backfillScope.endBlock, + config: indexingConfig, + } satisfies Unvalidated); +} + +/** + * Build Chain Indexing Status Snapshots for all indexed chains. + * + * @param chainsIndexingMetadata - A map of chain IDs to their complete + * indexing metadata. + * + * @returns A map of chain IDs to their chain indexing status snapshots. + */ +export function buildChainStatusSnapshots( + chainsIndexingMetadata: Map, +): Map { + const chainStatusSnapshots = new Map(); + + // Build chain indexing status snapshot for each indexed chain. + for (const [chainId, metadata] of chainsIndexingMetadata) { + const chainStatusSnapshot = buildChainIndexingStatusSnapshot(metadata); + + chainStatusSnapshots.set(chainId, chainStatusSnapshot); + } + + return chainStatusSnapshots; +} diff --git a/apps/ensindexer/src/lib/indexing-status-builder/cross-chain-indexing-status-snapshot.test.ts b/apps/ensindexer/src/lib/indexing-status-builder/cross-chain-indexing-status-snapshot.test.ts new file mode 100644 index 000000000..7f14ab657 --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/cross-chain-indexing-status-snapshot.test.ts @@ -0,0 +1,400 @@ +import { describe, expect, it } from "vitest"; + +import { + ChainIndexingConfigTypeIds, + ChainIndexingStatusIds, + type ChainIndexingStatusSnapshotBackfill, + type ChainIndexingStatusSnapshotCompleted, + type ChainIndexingStatusSnapshotFollowing, + type ChainIndexingStatusSnapshotQueued, + CrossChainIndexingStrategyIds, + OmnichainIndexingStatusIds, + type OmnichainIndexingStatusSnapshotBackfill, + type OmnichainIndexingStatusSnapshotCompleted, + type OmnichainIndexingStatusSnapshotFollowing, + type OmnichainIndexingStatusSnapshotUnstarted, +} from "@ensnode/ensnode-sdk"; +import type { UnixTimestamp } from "@ensnode/ponder-sdk"; + +import { earliestBlockRef, laterBlockRef, latestBlockRef } from "./block-refs.mock"; +import { buildCrossChainIndexingStatusSnapshotOmnichain } from "./cross-chain-indexing-status-snapshot"; + +describe("cross-chain-indexing-status-snapshot", () => { + describe("buildCrossChainIndexingStatusSnapshotOmnichain", () => { + it("builds snapshot from omnichain backfill snapshot", () => { + // arrange + const snapshotTime = latestBlockRef.timestamp as UnixTimestamp; + const cursor = laterBlockRef.timestamp; + + const chains = new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Backfill, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + backfillEndBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotBackfill, + ], + ]); + + const omnichainSnapshot = { + omnichainStatus: OmnichainIndexingStatusIds.Backfill, + chains, + omnichainIndexingCursor: cursor, + } satisfies OmnichainIndexingStatusSnapshotBackfill; + + // act + const result = buildCrossChainIndexingStatusSnapshotOmnichain( + omnichainSnapshot, + snapshotTime, + ); + + // assert + expect(result).toStrictEqual({ + strategy: CrossChainIndexingStrategyIds.Omnichain, + slowestChainIndexingCursor: cursor, + snapshotTime, + omnichainSnapshot: { + omnichainStatus: OmnichainIndexingStatusIds.Backfill, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Backfill, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + backfillEndBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotBackfill, + ], + ]), + omnichainIndexingCursor: cursor, + }, + }); + }); + + it("builds snapshot from omnichain completed snapshot", () => { + // arrange + const snapshotTime = latestBlockRef.timestamp as UnixTimestamp; + const cursor = latestBlockRef.timestamp; + + const chains = new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Completed, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotCompleted, + ], + ]); + + const omnichainSnapshot = { + omnichainStatus: OmnichainIndexingStatusIds.Completed, + chains, + omnichainIndexingCursor: cursor, + } satisfies OmnichainIndexingStatusSnapshotCompleted; + + // act + const result = buildCrossChainIndexingStatusSnapshotOmnichain( + omnichainSnapshot, + snapshotTime, + ); + + // assert + expect(result).toStrictEqual({ + strategy: CrossChainIndexingStrategyIds.Omnichain, + slowestChainIndexingCursor: cursor, + snapshotTime, + omnichainSnapshot: { + omnichainStatus: OmnichainIndexingStatusIds.Completed, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Completed, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: latestBlockRef, + }, + ], + ]), + omnichainIndexingCursor: cursor, + }, + }); + }); + + it("builds snapshot from omnichain following snapshot", () => { + // arrange + const snapshotTime = latestBlockRef.timestamp as UnixTimestamp; + const cursor = laterBlockRef.timestamp; + + const chains = new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Following, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: earliestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + latestKnownBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotFollowing, + ], + ]); + + const omnichainSnapshot = { + omnichainStatus: OmnichainIndexingStatusIds.Following, + chains, + omnichainIndexingCursor: cursor, + } satisfies OmnichainIndexingStatusSnapshotFollowing; + + // act + const result = buildCrossChainIndexingStatusSnapshotOmnichain( + omnichainSnapshot, + snapshotTime, + ); + + // assert + expect(result).toStrictEqual({ + strategy: CrossChainIndexingStrategyIds.Omnichain, + slowestChainIndexingCursor: cursor, + snapshotTime, + omnichainSnapshot: { + omnichainStatus: OmnichainIndexingStatusIds.Following, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Following, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: earliestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + latestKnownBlock: latestBlockRef, + }, + ], + ]), + omnichainIndexingCursor: cursor, + }, + }); + }); + + it("builds snapshot from omnichain unstarted snapshot", () => { + // arrange + const cursor = earliestBlockRef.timestamp - 1; + // snapshotTime must be >= highest known block timestamp + // for unstarted with queued chain, this is endBlock timestamp + const snapshotTime = latestBlockRef.timestamp as UnixTimestamp; + + const chains = new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Queued, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + } satisfies ChainIndexingStatusSnapshotQueued, + ], + ]); + + const omnichainSnapshot = { + omnichainStatus: OmnichainIndexingStatusIds.Unstarted, + chains, + omnichainIndexingCursor: cursor, + } satisfies OmnichainIndexingStatusSnapshotUnstarted; + + // act + const result = buildCrossChainIndexingStatusSnapshotOmnichain( + omnichainSnapshot, + snapshotTime, + ); + + // assert + expect(result).toStrictEqual({ + strategy: CrossChainIndexingStrategyIds.Omnichain, + slowestChainIndexingCursor: cursor, + snapshotTime, + omnichainSnapshot: { + omnichainStatus: OmnichainIndexingStatusIds.Unstarted, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Queued, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + }, + ], + ]), + omnichainIndexingCursor: cursor, + }, + }); + }); + + it("includes correct slowestChainIndexingCursor value", () => { + // arrange + const cursor = latestBlockRef.timestamp; + const snapshotTime = latestBlockRef.timestamp as UnixTimestamp; + + const chains = new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Completed, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotCompleted, + ], + ]); + + const omnichainSnapshot = { + omnichainStatus: OmnichainIndexingStatusIds.Completed, + chains, + omnichainIndexingCursor: cursor, + } satisfies OmnichainIndexingStatusSnapshotCompleted; + + // act + const result = buildCrossChainIndexingStatusSnapshotOmnichain( + omnichainSnapshot, + snapshotTime, + ); + + // assert + expect(result).toStrictEqual({ + strategy: CrossChainIndexingStrategyIds.Omnichain, + slowestChainIndexingCursor: cursor, + snapshotTime, + omnichainSnapshot: { + omnichainStatus: OmnichainIndexingStatusIds.Completed, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Completed, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: latestBlockRef, + }, + ], + ]), + omnichainIndexingCursor: cursor, + }, + }); + }); + + it("preserves nested omnichainSnapshot structure", () => { + // arrange + const cursor = laterBlockRef.timestamp; + const snapshotTime = latestBlockRef.timestamp as UnixTimestamp; + + const chains = new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Following, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: earliestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + latestKnownBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotFollowing, + ], + [ + 8453, + { + chainStatus: ChainIndexingStatusIds.Backfill, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + backfillEndBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotBackfill, + ], + ]); + + const omnichainSnapshot = { + omnichainStatus: OmnichainIndexingStatusIds.Following, + chains, + omnichainIndexingCursor: cursor, + } satisfies OmnichainIndexingStatusSnapshotFollowing; + + // act + const result = buildCrossChainIndexingStatusSnapshotOmnichain( + omnichainSnapshot, + snapshotTime, + ); + + // assert + expect(result).toStrictEqual({ + strategy: CrossChainIndexingStrategyIds.Omnichain, + slowestChainIndexingCursor: cursor, + snapshotTime, + omnichainSnapshot: { + omnichainStatus: OmnichainIndexingStatusIds.Following, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Following, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: earliestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + latestKnownBlock: latestBlockRef, + }, + ], + [ + 8453, + { + chainStatus: ChainIndexingStatusIds.Backfill, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: laterBlockRef, + backfillEndBlock: latestBlockRef, + }, + ], + ]), + omnichainIndexingCursor: cursor, + }, + }); + }); + }); +}); diff --git a/apps/ensindexer/src/lib/indexing-status-builder/cross-chain-indexing-status-snapshot.ts b/apps/ensindexer/src/lib/indexing-status-builder/cross-chain-indexing-status-snapshot.ts new file mode 100644 index 000000000..eecac3bf2 --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/cross-chain-indexing-status-snapshot.ts @@ -0,0 +1,20 @@ +import { + type CrossChainIndexingStatusSnapshotOmnichain, + CrossChainIndexingStrategyIds, + type OmnichainIndexingStatusSnapshot, + type Unvalidated, + validateCrossChainIndexingStatusSnapshot, +} from "@ensnode/ensnode-sdk"; +import type { UnixTimestamp } from "@ensnode/ponder-sdk"; + +export function buildCrossChainIndexingStatusSnapshotOmnichain( + omnichainSnapshot: OmnichainIndexingStatusSnapshot, + snapshotTime: UnixTimestamp, +): CrossChainIndexingStatusSnapshotOmnichain { + return validateCrossChainIndexingStatusSnapshot({ + strategy: CrossChainIndexingStrategyIds.Omnichain, + slowestChainIndexingCursor: omnichainSnapshot.omnichainIndexingCursor, + snapshotTime, + omnichainSnapshot, + } satisfies Unvalidated); +} diff --git a/apps/ensindexer/src/lib/indexing-status-builder/omnichain-indexing-status-snapshot.test.ts b/apps/ensindexer/src/lib/indexing-status-builder/omnichain-indexing-status-snapshot.test.ts new file mode 100644 index 000000000..42889db9c --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/omnichain-indexing-status-snapshot.test.ts @@ -0,0 +1,300 @@ +import { assert, describe, expect, it } from "vitest"; + +import { + ChainIndexingConfigTypeIds, + ChainIndexingStatusIds, + type ChainIndexingStatusSnapshotBackfill, + type ChainIndexingStatusSnapshotCompleted, + type ChainIndexingStatusSnapshotFollowing, + type ChainIndexingStatusSnapshotQueued, + createIndexingConfig, + OmnichainIndexingStatusIds, +} from "@ensnode/ensnode-sdk"; +import { type BlockRef, ChainIndexingStates } from "@ensnode/ponder-sdk"; + +import { + earlierBlockRef, + earliestBlockRef, + laterBlockRef, + latestBlockRef, +} from "./block-refs.mock"; +import { buildChainIndexingMetadataMock } from "./chain-indexing-metadata.mock"; +import { buildOmnichainIndexingStatusSnapshot } from "./omnichain-indexing-status-snapshot"; + +describe("OmnichainIndexingStatusSnapshot", () => { + describe("buildOmnichainIndexingStatusSnapshot", () => { + it("returns Unstarted status when all chains are Queued", () => { + // arrange + const metadata = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, null), + checkpointBlock: earliestBlockRef, + state: ChainIndexingStates.Historical, + latestSyncedBlock: laterBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const chainsMetadata = new Map([[1, metadata]]); + + // act + const result = buildOmnichainIndexingStatusSnapshot(chainsMetadata); + + // assert + expect(result).toStrictEqual({ + omnichainStatus: OmnichainIndexingStatusIds.Unstarted, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Queued, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: earliestBlockRef, + }, + } satisfies ChainIndexingStatusSnapshotQueued, + ], + ]), + omnichainIndexingCursor: earliestBlockRef.timestamp - 1, + }); + }); + + it("returns Backfill status when at least one chain is Backfill and none are Following", () => { + // arrange + const metadataQueued = buildChainIndexingMetadataMock({ + config: createIndexingConfig(laterBlockRef, null), + checkpointBlock: laterBlockRef, + state: ChainIndexingStates.Historical, + latestSyncedBlock: latestBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const metadataBackfill = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, null), + checkpointBlock: earlierBlockRef, + state: ChainIndexingStates.Historical, + latestSyncedBlock: laterBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const chainsMetadata = new Map([ + [1, metadataQueued], + [8453, metadataBackfill], + ]); + + // act + const result = buildOmnichainIndexingStatusSnapshot(chainsMetadata); + + // assert + expect(result).toStrictEqual({ + omnichainStatus: OmnichainIndexingStatusIds.Backfill, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Queued, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: metadataQueued.indexingConfig.startBlock, + }, + } satisfies ChainIndexingStatusSnapshotQueued, + ], + [ + 8453, + { + chainStatus: ChainIndexingStatusIds.Backfill, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: metadataBackfill.indexingConfig.startBlock, + }, + latestIndexedBlock: metadataBackfill.indexingStatus.checkpointBlock, + backfillEndBlock: metadataBackfill.backfillScope.endBlock, + } satisfies ChainIndexingStatusSnapshotBackfill, + ], + ]), + omnichainIndexingCursor: metadataBackfill.indexingStatus.checkpointBlock.timestamp, + }); + }); + + it("returns Following status when at least one chain is Following", () => { + // arrange + + const metadataBackfill = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earlierBlockRef, null), + checkpointBlock: laterBlockRef, + state: ChainIndexingStates.Historical, + latestSyncedBlock: latestBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const metadataFollowing = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, null), + checkpointBlock: laterBlockRef, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: latestBlockRef, + backfillEndBlock: earlierBlockRef, + }); + + const chainsMetadata = new Map([ + [1, metadataBackfill], + [8453, metadataFollowing], + ]); + + // act + const result = buildOmnichainIndexingStatusSnapshot(chainsMetadata); + + // assert + assert( + metadataFollowing.indexingMetrics.state === ChainIndexingStates.Realtime, + "Expected Realtime state for Following chain in test setup", + ); + + expect(result).toStrictEqual({ + omnichainStatus: OmnichainIndexingStatusIds.Following, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Backfill, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: metadataBackfill.indexingConfig.startBlock, + }, + latestIndexedBlock: metadataBackfill.indexingStatus.checkpointBlock, + backfillEndBlock: metadataBackfill.backfillScope.endBlock, + } satisfies ChainIndexingStatusSnapshotBackfill, + ], + [ + 8453, + { + chainStatus: ChainIndexingStatusIds.Following, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: metadataFollowing.indexingConfig.startBlock, + }, + latestIndexedBlock: metadataFollowing.indexingStatus.checkpointBlock, + latestKnownBlock: metadataFollowing.indexingMetrics.latestSyncedBlock, + } satisfies ChainIndexingStatusSnapshotFollowing, + ], + ]), + omnichainIndexingCursor: metadataFollowing.indexingStatus.checkpointBlock.timestamp, + }); + }); + + it("returns Completed status when all chains are Completed", () => { + // arrange + const metadataCompleted = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, latestBlockRef), + checkpointBlock: latestBlockRef, + state: ChainIndexingStates.Completed, + latestSyncedBlock: latestBlockRef, + backfillEndBlock: latestBlockRef, + }); + + const chainsMetadata = new Map([[1, metadataCompleted]]); + + // act + const result = buildOmnichainIndexingStatusSnapshot(chainsMetadata); + + // assert + expect(result).toStrictEqual({ + omnichainStatus: OmnichainIndexingStatusIds.Completed, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Completed, + config: { + configType: ChainIndexingConfigTypeIds.Definite, + startBlock: earliestBlockRef, + endBlock: latestBlockRef, + }, + latestIndexedBlock: latestBlockRef, + } satisfies ChainIndexingStatusSnapshotCompleted, + ], + ]), + omnichainIndexingCursor: latestBlockRef.timestamp, + }); + }); + + it("correctly calculates omnichainIndexingCursor with mixed status chains", () => { + // arrange + const evenLaterBlockRef: BlockRef = { + timestamp: latestBlockRef.timestamp + 1000, + number: latestBlockRef.number + 1000, + }; + + const metadata1 = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earliestBlockRef, null), + checkpointBlock: laterBlockRef, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: evenLaterBlockRef, + backfillEndBlock: earlierBlockRef, + }); + + const metadata2 = buildChainIndexingMetadataMock({ + config: createIndexingConfig(earlierBlockRef, null), + checkpointBlock: laterBlockRef, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: latestBlockRef, + backfillEndBlock: laterBlockRef, + }); + + const chainsMetadata = new Map([ + [1, metadata1], + [8453, metadata2], + ]); + + // act + const result = buildOmnichainIndexingStatusSnapshot(chainsMetadata); + + // assert + assert( + metadata1.indexingMetrics.state === ChainIndexingStates.Realtime, + "Expected Realtime state for Following chain in test setup", + ); + assert( + metadata2.indexingMetrics.state === ChainIndexingStates.Realtime, + "Expected Realtime state for Following chain in test setup", + ); + expect(result).toStrictEqual({ + omnichainStatus: OmnichainIndexingStatusIds.Following, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Following, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: metadata1.indexingConfig.startBlock, + }, + latestIndexedBlock: metadata1.indexingStatus.checkpointBlock, + latestKnownBlock: metadata1.indexingMetrics.latestSyncedBlock, + } satisfies ChainIndexingStatusSnapshotFollowing, + ], + [ + 8453, + { + chainStatus: ChainIndexingStatusIds.Following, + config: { + configType: ChainIndexingConfigTypeIds.Indefinite, + startBlock: metadata2.indexingConfig.startBlock, + }, + latestIndexedBlock: metadata2.indexingStatus.checkpointBlock, + latestKnownBlock: metadata2.indexingMetrics.latestSyncedBlock, + } satisfies ChainIndexingStatusSnapshotFollowing, + ], + ]), + omnichainIndexingCursor: laterBlockRef.timestamp, + }); + }); + + it("throws an error when no chains metadata is provided", () => { + // arrange + const chainsMetadata = new Map(); + + // act & assert + expect(() => buildOmnichainIndexingStatusSnapshot(chainsMetadata)).toThrowError( + /At least one chain's indexing metadata is required to build an OmnichainIndexingStatusSnapshot/, + ); + }); + }); +}); diff --git a/apps/ensindexer/src/lib/indexing-status-builder/omnichain-indexing-status-snapshot.ts b/apps/ensindexer/src/lib/indexing-status-builder/omnichain-indexing-status-snapshot.ts new file mode 100644 index 000000000..81712062a --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-status-builder/omnichain-indexing-status-snapshot.ts @@ -0,0 +1,88 @@ +import { + type ChainIndexingStatusSnapshotCompleted, + type ChainIndexingStatusSnapshotForOmnichainIndexingStatusSnapshotBackfill, + type ChainIndexingStatusSnapshotQueued, + getOmnichainIndexingCursor, + getOmnichainIndexingStatus, + OmnichainIndexingStatusIds, + type OmnichainIndexingStatusSnapshot, + type OmnichainIndexingStatusSnapshotBackfill, + type OmnichainIndexingStatusSnapshotCompleted, + type OmnichainIndexingStatusSnapshotFollowing, + type OmnichainIndexingStatusSnapshotUnstarted, + type Unvalidated, + validateOmnichainIndexingStatusSnapshot, +} from "@ensnode/ensnode-sdk"; +import type { ChainId } from "@ensnode/ponder-sdk"; + +import type { ChainIndexingMetadata } from "./chain-indexing-metadata"; +import { buildChainStatusSnapshots } from "./chain-indexing-status-snapshot"; + +/** + * Build an omnichain indexing status snapshot from per-chain metadata. + * + * @param chainsIndexingMetadata - A map of chain IDs to their complete indexing + * metadata. Each entry contains the backfill scope, Ponder metrics, and status + * needed to determine that chain's indexing state. + * + * @returns The validated omnichain indexing status snapshot. + * @throws Error if the provided metadata is insufficient to determine the status + * snapshot for any chain, or if the resulting snapshot is invalid. + */ +export function buildOmnichainIndexingStatusSnapshot( + chainsIndexingMetadata: Map, +): OmnichainIndexingStatusSnapshot { + if (chainsIndexingMetadata.size === 0) { + throw new Error( + "At least one chain's indexing metadata is required to build an OmnichainIndexingStatusSnapshot", + ); + } + + const chainStatusSnapshots = buildChainStatusSnapshots(chainsIndexingMetadata); + + const chains = Array.from(chainStatusSnapshots.values()); + const omnichainStatus = getOmnichainIndexingStatus(chains); + const omnichainIndexingCursor = getOmnichainIndexingCursor(chains); + + switch (omnichainStatus) { + case OmnichainIndexingStatusIds.Unstarted: { + return validateOmnichainIndexingStatusSnapshot({ + omnichainStatus: OmnichainIndexingStatusIds.Unstarted, + chains: chainStatusSnapshots as Map< + ChainId, + Unvalidated + >, // narrowing the type here, will be validated in the following 'check' step + omnichainIndexingCursor, + } satisfies Unvalidated); + } + + case OmnichainIndexingStatusIds.Backfill: { + return validateOmnichainIndexingStatusSnapshot({ + omnichainStatus: OmnichainIndexingStatusIds.Backfill, + chains: chainStatusSnapshots as Map< + ChainId, + Unvalidated + >, // narrowing the type here, will be validated in the following 'check' step + omnichainIndexingCursor, + } satisfies Unvalidated); + } + + case OmnichainIndexingStatusIds.Completed: { + return validateOmnichainIndexingStatusSnapshot({ + omnichainStatus: OmnichainIndexingStatusIds.Completed, + chains: chainStatusSnapshots as Map< + ChainId, + Unvalidated + >, // narrowing the type here, will be validated in the following 'check' step + omnichainIndexingCursor, + } satisfies Unvalidated); + } + + case OmnichainIndexingStatusIds.Following: + return validateOmnichainIndexingStatusSnapshot({ + omnichainStatus: OmnichainIndexingStatusIds.Following, + chains: chainStatusSnapshots, + omnichainIndexingCursor, + } satisfies Unvalidated); + } +} diff --git a/packages/ponder-sdk/src/blocks.ts b/packages/ponder-sdk/src/blocks.ts index 05d985073..609c4e9f2 100644 --- a/packages/ponder-sdk/src/blocks.ts +++ b/packages/ponder-sdk/src/blocks.ts @@ -25,3 +25,70 @@ export const schemaBlockRef = z.object({ * Reference to a block. */ export type BlockRef = z.infer; + +/** + * Compare two {@link BlockRef} objects to check + * if blockA is before blockB. + */ +export function isBlockRefBefore(blockA: BlockRef, blockB: BlockRef) { + return blockA.number < blockB.number && blockA.timestamp < blockB.timestamp; +} + +/** + * Compare two {@link BlockRef} objects to check + * if blockA is equal to blockB. + */ +export function isBlockRefEqualTo(blockA: BlockRef, blockB: BlockRef) { + return blockA.number === blockB.number && blockA.timestamp === blockB.timestamp; +} + +/** + * Compare two {@link BlockRef} objects to check + * if blockA is before or equal to blockB. + */ +export function isBlockRefBeforeOrEqualTo(blockA: BlockRef, blockB: BlockRef) { + return isBlockRefBefore(blockA, blockB) || isBlockRefEqualTo(blockA, blockB); +} + +/** + * Block range + * + * Represents a range of blocks + */ +export interface Blockrange { + /** + * Start block number + * + * Guaranteed to be lower than `endBlock` when both are present. + */ + startBlock?: BlockNumber; + + /** + * End block number + * + * Guaranteed to be greater than `startBlock` when both are present. + */ + endBlock?: BlockNumber; +} + +/** + * Block range with required start block + * + * Represents a range of blocks where the start block is required and the end + * block is optional. + */ +export interface BlockrangeWithStartBlock { + /** + * Start block number + * + * Guaranteed to be lower than `endBlock` when both are present. + */ + startBlock: BlockNumber; + + /** + * End block number + * + * Guaranteed to be greater than `startBlock` when both are present. + */ + endBlock?: BlockNumber; +} diff --git a/packages/ponder-sdk/src/index.ts b/packages/ponder-sdk/src/index.ts index 683a3b2e3..6c161e332 100644 --- a/packages/ponder-sdk/src/index.ts +++ b/packages/ponder-sdk/src/index.ts @@ -1,3 +1,7 @@ +export * from "./blocks"; +export * from "./chains"; export * from "./client"; export * from "./indexing-metrics"; export * from "./indexing-status"; +export * from "./numbers"; +export * from "./time"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8107c35c0..4941663f9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -457,6 +457,9 @@ importers: '@ensnode/ponder-metadata': specifier: workspace:* version: link:../../packages/ponder-metadata + '@ensnode/ponder-sdk': + specifier: workspace:* + version: link:../../packages/ponder-sdk caip: specifier: 'catalog:' version: 1.1.1