diff --git a/.changeset/clean-rivers-buy.md b/.changeset/clean-rivers-buy.md new file mode 100644 index 0000000000..814feb846e --- /dev/null +++ b/.changeset/clean-rivers-buy.md @@ -0,0 +1,19 @@ +--- +"@ensnode/integration-test-env": minor +"@ensnode/ensnode-sdk": minor +"@ensnode/ensdb-sdk": minor +"ensindexer": minor +"ensapi": minor +--- + +Introduced `IndexingMetadataContext` data model, a single record type in ENSNode Metadata table replacing three separate record types (`ensdb_version`, `ensindexer_public_config`, `ensindexer_indexing_status`). Also, consolidated startup init into `initIndexingOnchainEvents()` for reliable execution on every ENSIndexer startup. + +**ensnode-sdk**: `EnsIndexerStackInfo` added as base type, `EnsNodeStackInfo` refactored to extend it. + +**ensdb-sdk**: For `EnsDbReader`, added following method: `getIndexingMetadataContext()`, `isHealthy()`, `isReady()`. For `EnsDbWriter`, added `upsertIndexingMetadataContext()` method. Old per-record read/write methods removed. `EnsNodeMetadataKeys` reduced to single `IndexingMetadataContext` key. + +**ensindexer**: `IndexingMetadataContextBuilder` and `StackInfoBuilder` added. `EnsDbWriterWorker` simplified to single recurring task. HTTP `/config` and `/indexing-status` endpoints now read from in-memory builders instead of ENSDb. `initializeIndexingSetup`/`initializeIndexingActivation` replaced by `initIndexingOnchainEvents`. + +**ensapi**: `indexing-status.cache` and `stack-info.cache` updated to consume `IndexingMetadataContext`. Config schema updated to fetch `EnsIndexerPublicConfig` from `EnsNodeStackInfo`. + +**integration-test-env**: `pollIndexingStatus` updated to use `getIndexingMetadataContext()`. diff --git a/apps/ensapi/src/cache/indexing-status.cache.ts b/apps/ensapi/src/cache/indexing-status.cache.ts index 8ae9562f29..51453f1534 100644 --- a/apps/ensapi/src/cache/indexing-status.cache.ts +++ b/apps/ensapi/src/cache/indexing-status.cache.ts @@ -1,5 +1,9 @@ import { EnsNodeMetadataKeys } from "@ensnode/ensdb-sdk"; -import { type CrossChainIndexingStatusSnapshot, SWRCache } from "@ensnode/ensnode-sdk"; +import { + type CrossChainIndexingStatusSnapshot, + IndexingMetadataContextStatusCodes, + SWRCache, +} from "@ensnode/ensnode-sdk"; import { ensDbClient } from "@/lib/ensdb/singleton"; import { lazyProxy } from "@/lib/lazy"; @@ -7,44 +11,57 @@ import { makeLogger } from "@/lib/logger"; const logger = makeLogger("indexing-status.cache"); +export type IndexingStatusCache = SWRCache; + // lazyProxy defers construction until first use so that this module can be // imported without env vars being present (e.g. during OpenAPI generation). // SWRCache with proactivelyInitialize:true starts background polling immediately // on construction, which would trigger ensDbClient before env vars are available. -export const indexingStatusCache = lazyProxy>( +/** + * Cache for {@link CrossChainIndexingStatusSnapshot}, which is loaded + * from ENSDb on demand. The cached value is expected to be updated + * very frequently, following the update frequency of + * {@link IndexingMetadataContextInitialized.indexingStatus} in ENSDb. + * Therefore, the cache is configured with a very short TTL and + * proactive revalidation interval to ensure that the cached value is + * as fresh as possible. + */ +export const indexingStatusCache = lazyProxy( () => new SWRCache({ - fn: async (_cachedResult) => - ensDbClient - .getIndexingStatusSnapshot() // get the latest indexing status snapshot - .then((snapshot) => { - if (snapshot === undefined) { - // An indexing status snapshot has not been found in ENSDb yet. - // This might happen during application startup, i.e. when ENSDb - // has not yet been populated with the first snapshot. - // Therefore, throw an error to trigger the subsequent `.catch` handler. - throw new Error("Indexing Status snapshot not found in ENSDb yet."); - } + fn: async function loadIndexingStatusSnapshot() { + try { + const indexingMetadataContext = await ensDbClient.getIndexingMetadataContext(); + + if ( + indexingMetadataContext.statusCode !== IndexingMetadataContextStatusCodes.Initialized + ) { + // The IndexingMetadataContext has not been initialized in ENSDb yet. + // This might happen during application startup, i.e. when ENSDb + // has not yet been populated with the IndexingMetadataContext record. + // Therefore, throw an error to trigger the subsequent catch handler. + throw new Error("Indexing Metadata Context was uninitialized in ENSDb."); + } - // The indexing status snapshot has been fetched and successfully validated for caching. - // Therefore, return it so that this current invocation of `readCache` will: - // - Replace the currently cached value (if any) with this new value. - // - Return this non-null value. - return snapshot; - }) - .catch((error) => { - // Either the indexing status snapshot fetch failed, or the indexing status snapshot was not found in ENSDb yet. - // Therefore, throw an error so that this current invocation of `readCache` will: - // - Reject the newly fetched response (if any) such that it won't be cached. - // - Return the most recently cached value from prior invocations, or `null` if no prior invocation successfully cached a value. - logger.error( - error, - `Error occurred while loading Indexing Status snapshot record from ENSNode Metadata table in ENSDb. ` + - `Where clause applied: ("ensIndexerSchemaName" = "${ensDbClient.ensIndexerSchemaName}", "key" = "${EnsNodeMetadataKeys.EnsIndexerIndexingStatus}"). ` + - `The cached indexing status snapshot (if any) will not be updated.`, - ); - throw error; - }), + // The CrossChainIndexingStatusSnapshot has been successfully loaded for caching. + // Therefore, return it so that this current invocation of `readCache` will: + // - Replace the currently cached value (if any) with this new value. + // - Return this non-null value. + return indexingMetadataContext.indexingStatus; + } catch (error) { + // IndexingMetadataContext was uninitialized in ENSDb. + // Therefore, throw an error so that this current invocation of `readCache` will: + // - Reject the newly fetched response (if any) such that it won't be cached. + // - Return the most recently cached value from prior invocations, or `null` if no prior invocation successfully cached a value. + logger.error( + error, + `Error occurred while loading Indexing Metadata Context record from ENSNode Metadata table in ENSDb. ` + + `Where clause applied: ("ensIndexerSchemaName" = "${ensDbClient.ensIndexerSchemaName}", "key" = "${EnsNodeMetadataKeys.IndexingMetadataContext}"). ` + + `The cached indexing status snapshot (if any) will not be updated.`, + ); + throw error; + } + }, // We need to refresh the indexing status cache very frequently. // ENSDb won't have issues handling this frequency of queries. ttl: 1, // 1 second diff --git a/apps/ensapi/src/cache/stack-info.cache.ts b/apps/ensapi/src/cache/stack-info.cache.ts index 3a94a38b5c..243f9738c8 100644 --- a/apps/ensapi/src/cache/stack-info.cache.ts +++ b/apps/ensapi/src/cache/stack-info.cache.ts @@ -2,76 +2,91 @@ import config from "@/config"; import { minutesToSeconds } from "date-fns"; +import { EnsNodeMetadataKeys } from "@ensnode/ensdb-sdk"; import { buildEnsNodeStackInfo, - type CachedResult, type EnsNodeStackInfo, + type IndexingMetadataContextInitialized, + IndexingMetadataContextStatusCodes, SWRCache, } from "@ensnode/ensnode-sdk"; import { buildEnsApiPublicConfig } from "@/config/config.schema"; import { ensDbClient } from "@/lib/ensdb/singleton"; import { lazyProxy } from "@/lib/lazy"; +import logger from "@/lib/logger"; -/** - * Loads the ENSNode stack info, either from cache if available, - * or by building it from the public configs of ENSApi and ENSDb. - * - * The ENSNode Stack Info object is considered immutable for - * the lifecycle of an ENSApi process instance, so once it is successfully - * loaded, it will be cached indefinitely. - */ -async function loadEnsNodeStackInfo( - cachedResult?: CachedResult, -): Promise { - if (cachedResult && !(cachedResult.result instanceof Error)) { - return cachedResult.result; - } - - const ensApiPublicConfig = buildEnsApiPublicConfig(config); - const ensDbPublicConfig = await ensDbClient.buildEnsDbPublicConfig(); - const ensIndexerPublicConfig = ensApiPublicConfig.ensIndexerPublicConfig; - const ensRainbowPublicConfig = ensIndexerPublicConfig.ensRainbowPublicConfig; - - return buildEnsNodeStackInfo( - ensApiPublicConfig, - ensDbPublicConfig, - ensIndexerPublicConfig, - ensRainbowPublicConfig, - ); -} +export type EnsNodeStackInfoCache = SWRCache; // lazyProxy defers construction until first use so that this module can be // imported without env vars being present (e.g. during OpenAPI generation). // SWRCache with proactivelyInitialize:true starts background polling immediately // on construction, which would trigger ensDbClient before env vars are available. /** - * Cache for ENSNode stack info - * Once successfully loaded, the ENSNode Stack Info is cached indefinitely and - * never revalidated. This ensures the JSON is only fetched once during - * the application lifecycle. + * Cache for {@link EnsNodeStackInfo}, which is loaded from ENSDb on demand. + * Once successfully loaded, the {@link EnsNodeStackInfo} is cached and kept up-to-date + * by proactive revalidation, since the {@link EnsNodeStackInfo} might change during + * the lifecycle of the ENSApi instance, for example, when + * {@link IndexingMetadataContextInitialized.stackInfo} is updated in ENSDb. + * This is unlikely to happen at all, and if it does happen, it is likely to be + * very infrequent. However, proactive revalidation ensures that if such changes do happen, + * the cached value will be updated in a reasonable time frame without requiring + * a restart of the ENSApi application. * * Configuration: - * - ttl: Infinity - Never expires once cached - * - errorTtl: 1 minute - If loading fails, retry on next access after 1 minute - * - proactiveRevalidationInterval: undefined - No proactive revalidation + * - ttl: 1 minute - Allow cached value to be fresh for up to 1 minute. + * - errorTtl: 1 minute - If loading fails, retry on next access after 1 minute. + * - proactiveRevalidationInterval: 5 minutes - Refresh the cached value every 5 minutes. * - proactivelyInitialize: true - Load immediately on startup */ -export const stackInfoCache = lazyProxy( +export const stackInfoCache = lazyProxy( () => - /** - * Cache for ENSNode stack info - * - * Once initialized successfully, this cache will always return - * the same stack info for the lifecycle of the ENSApi instance. - * - * If initialization fails, it will keep retrying on access until it succeeds, which is desirable because the stack info is critical for the functioning of the application and we want to recover from transient initialization failures without requiring a restart. - */ new SWRCache({ - fn: loadEnsNodeStackInfo, - ttl: Number.POSITIVE_INFINITY, + fn: async function loadEnsNodeStackInfo() { + try { + const indexingMetadataContext = await ensDbClient.getIndexingMetadataContext(); + + if ( + indexingMetadataContext.statusCode !== IndexingMetadataContextStatusCodes.Initialized + ) { + // The IndexingMetadataContext has not been initialized in ENSDb yet. + // This might happen during application startup, i.e. when ENSDb + // has not yet been populated with the IndexingMetadataContext record. + // Therefore, throw an error to trigger the subsequent catch handler. + throw new Error("Indexing Metadata Context was uninitialized in ENSDb."); + } + + const ensIndexerStackInfo = indexingMetadataContext.stackInfo; + const ensNodeStackInfo = buildEnsNodeStackInfo( + buildEnsApiPublicConfig(config, ensIndexerStackInfo.ensIndexer), + ensIndexerStackInfo.ensDb, + ensIndexerStackInfo.ensIndexer, + ensIndexerStackInfo.ensRainbow, + ); + + // The EnsNodeStackInfo has been successfully built for caching. + // Therefore, return it so that this current invocation of `readCache` will: + // - Replace the currently cached value (if any) with this new value. + // - Return this non-null value. + return ensNodeStackInfo; + } catch (error) { + // IndexingMetadataContext was uninitialized in ENSDb. + // Therefore, throw an error so that this current invocation of `readCache` will: + // - Reject the newly fetched response (if any) such that it won't be cached. + // - Return the most recently cached value from prior invocations, or `null` if no prior invocation successfully cached a value. + logger.error( + error, + `Error occurred while loading Indexing Metadata Context record from ENSNode Metadata table in ENSDb. ` + + `Where clause applied: ("ensIndexerSchemaName" = "${ensDbClient.ensIndexerSchemaName}", "key" = "${EnsNodeMetadataKeys.IndexingMetadataContext}"). ` + + `The cached EnsNodeStackInfo object (if any) will not be updated.`, + ); + + throw error; + } + }, + ttl: minutesToSeconds(1), errorTtl: minutesToSeconds(1), - proactiveRevalidationInterval: undefined, + proactiveRevalidationInterval: minutesToSeconds(5), proactivelyInitialize: true, }), ); diff --git a/apps/ensapi/src/config/config.schema.mock.ts b/apps/ensapi/src/config/config.schema.mock.ts new file mode 100644 index 0000000000..7425c5ed6e --- /dev/null +++ b/apps/ensapi/src/config/config.schema.mock.ts @@ -0,0 +1,108 @@ +import packageJson from "@/../package.json" with { type: "json" }; + +import { + ChainIndexingStatusIds, + CrossChainIndexingStrategyIds, + deserializeIndexingMetadataContext, + type EnsRainbowPublicConfig, + type IndexingMetadataContextInitialized, + IndexingMetadataContextStatusCodes, + OmnichainIndexingStatusIds, + PluginName, + RangeTypeIds, + type SerializedCrossChainIndexingStatusSnapshot, + type SerializedEnsDbPublicConfig, + type SerializedEnsIndexerPublicConfig, + type SerializedEnsIndexerStackInfo, + type SerializedIndexingMetadataContextInitialized, +} from "@ensnode/ensnode-sdk"; + +import type { EnsApiEnvironment } from "@/config/environment"; + +export const VALID_RPC_URL = "https://eth-sepolia.g.alchemy.com/v2/1234"; + +export const ENSDB_PUBLIC_CONFIG = { + versionInfo: { + postgresql: "17.4", + }, +} satisfies SerializedEnsDbPublicConfig; + +export const ENSINDEXER_PUBLIC_CONFIG = { + namespace: "mainnet", + ensIndexerSchemaName: "ensindexer_0", + ensRainbowPublicConfig: { + serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, + versionInfo: { + ensRainbow: packageJson.version, + }, + }, + indexedChainIds: [1], + isSubgraphCompatible: false, + clientLabelSet: { labelSetId: "subgraph", labelSetVersion: 0 }, + plugins: [PluginName.Subgraph], + versionInfo: { + ensDb: packageJson.version, + ensIndexer: packageJson.version, + ensNormalize: "1.11.1", + ponder: "0.8.0", + }, +} satisfies SerializedEnsIndexerPublicConfig; + +const ENSRAINBOW_PUBLIC_CONFIG = { + serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, + versionInfo: { + ensRainbow: packageJson.version, + }, +} satisfies EnsRainbowPublicConfig; + +export const INDEXING_STATUS = { + strategy: CrossChainIndexingStrategyIds.Omnichain, + slowestChainIndexingCursor: 1777147427, + snapshotTime: 1777147440, + omnichainSnapshot: { + omnichainStatus: OmnichainIndexingStatusIds.Following, + chains: { + "1": { + chainStatus: ChainIndexingStatusIds.Following, + config: { + rangeType: RangeTypeIds.LeftBounded, + startBlock: { + timestamp: 1489165544, + number: 3327417, + }, + }, + latestIndexedBlock: { + timestamp: 1777147427, + number: 24959286, + }, + latestKnownBlock: { + timestamp: 1777147427, + number: 24959286, + }, + }, + }, + omnichainIndexingCursor: 1777147427, + }, +} satisfies SerializedCrossChainIndexingStatusSnapshot; + +export const ENSINDEXER_STACK_INFO = { + ensDb: ENSDB_PUBLIC_CONFIG, + ensIndexer: ENSINDEXER_PUBLIC_CONFIG, + ensRainbow: ENSRAINBOW_PUBLIC_CONFIG, +} satisfies SerializedEnsIndexerStackInfo; + +export const INDEXING_METADATA_CONTEXT = { + statusCode: IndexingMetadataContextStatusCodes.Initialized, + indexingStatus: INDEXING_STATUS, + stackInfo: ENSINDEXER_STACK_INFO, +} satisfies SerializedIndexingMetadataContextInitialized; + +export const indexingMetadataContextInitialized = deserializeIndexingMetadataContext( + INDEXING_METADATA_CONTEXT, +) as IndexingMetadataContextInitialized; + +export const BASE_ENV = { + ENSDB_URL: "postgresql://user:password@localhost:5432/mydb", + ENSINDEXER_SCHEMA_NAME: "ensindexer_0", + RPC_URL_1: VALID_RPC_URL, +} satisfies EnsApiEnvironment; diff --git a/apps/ensapi/src/config/config.schema.test.ts b/apps/ensapi/src/config/config.schema.test.ts index 4f1e9493c1..a29ab2bf74 100644 --- a/apps/ensapi/src/config/config.schema.test.ts +++ b/apps/ensapi/src/config/config.schema.test.ts @@ -1,13 +1,12 @@ -import packageJson from "@/../package.json" with { type: "json" }; - import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { type ENSIndexerPublicConfig, PluginName } from "@ensnode/ensnode-sdk"; import type { RpcConfig } from "@ensnode/ensnode-sdk/internal"; +import { ensApiVersionInfo } from "@/lib/version-info"; + vi.mock("@/lib/ensdb/singleton", () => ({ ensDbClient: { - getEnsIndexerPublicConfig: vi.fn(async () => ENSINDEXER_PUBLIC_CONFIG), + getIndexingMetadataContext: vi.fn(async () => indexingMetadataContextInitialized), }, })); @@ -19,10 +18,13 @@ vi.mock("@/config/ensdb-config", () => ({ })); import { buildConfigFromEnvironment, buildEnsApiPublicConfig } from "@/config/config.schema"; +import { + BASE_ENV, + indexingMetadataContextInitialized, + VALID_RPC_URL, +} from "@/config/config.schema.mock"; import { ENSApi_DEFAULT_PORT } from "@/config/defaults"; -import type { EnsApiEnvironment } from "@/config/environment"; import logger from "@/lib/logger"; -import { ensApiVersionInfo } from "@/lib/version-info"; vi.mock("@/lib/logger", () => ({ default: { @@ -31,44 +33,29 @@ vi.mock("@/lib/logger", () => ({ }, })); -const VALID_RPC_URL = "https://eth-sepolia.g.alchemy.com/v2/1234"; - -const BASE_ENV = { - ENSDB_URL: "postgresql://user:password@localhost:5432/mydb", - RPC_URL_1: VALID_RPC_URL, -} satisfies EnsApiEnvironment; - -const ENSINDEXER_PUBLIC_CONFIG = { - namespace: "mainnet", - ensIndexerSchemaName: "ensindexer_0", - ensRainbowPublicConfig: { - serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, - versionInfo: { - ensRainbow: packageJson.version, - }, - }, - indexedChainIds: new Set([1]), - isSubgraphCompatible: false, - clientLabelSet: { labelSetId: "subgraph", labelSetVersion: 0 }, - plugins: [PluginName.Subgraph], - versionInfo: { - ensDb: packageJson.version, - ensIndexer: packageJson.version, - ensNormalize: ensApiVersionInfo.ensNormalize, - ponder: "0.8.0", - }, -} satisfies ENSIndexerPublicConfig; +const mockProcessExit = () => + vi.spyOn(process, "exit").mockImplementation((() => { + throw new Error("process.exit"); + }) as never); describe("buildConfigFromEnvironment", () => { it("returns a valid config object using environment variables", async () => { - await expect(buildConfigFromEnvironment(BASE_ENV)).resolves.toStrictEqual({ + const exitSpy = mockProcessExit(); + + const { ensIndexer: ensIndexerPublicConfig } = indexingMetadataContextInitialized.stackInfo; + const config = await buildConfigFromEnvironment(BASE_ENV); + + expect(exitSpy).not.toHaveBeenCalled(); + exitSpy.mockRestore(); + + expect(config).toStrictEqual({ port: ENSApi_DEFAULT_PORT, ensDbUrl: BASE_ENV.ENSDB_URL, + ensIndexerSchemaName: BASE_ENV.ENSINDEXER_SCHEMA_NAME, theGraphApiKey: undefined, - ensIndexerPublicConfig: ENSINDEXER_PUBLIC_CONFIG, - namespace: ENSINDEXER_PUBLIC_CONFIG.namespace, - ensIndexerSchemaName: ENSINDEXER_PUBLIC_CONFIG.ensIndexerSchemaName, + ensIndexerPublicConfig, + namespace: ensIndexerPublicConfig.namespace, rpcConfigs: new Map([ [ 1, @@ -83,6 +70,7 @@ describe("buildConfigFromEnvironment", () => { }); it("parses REFERRAL_PROGRAM_EDITIONS as a URL object", async () => { + const exitSpy = mockProcessExit(); const editionsUrl = "https://example.com/editions.json"; const config = await buildConfigFromEnvironment({ @@ -90,75 +78,103 @@ describe("buildConfigFromEnvironment", () => { REFERRAL_PROGRAM_EDITIONS: editionsUrl, }); + expect(exitSpy).not.toHaveBeenCalled(); + exitSpy.mockRestore(); + expect(config.referralProgramEditionConfigSetUrl).toEqual(new URL(editionsUrl)); }); + it("includes theGraphApiKey when provided", async () => { + const exitSpy = mockProcessExit(); + + const config = await buildConfigFromEnvironment({ + ...BASE_ENV, + THEGRAPH_API_KEY: "my-api-key", + }); + + expect(exitSpy).not.toHaveBeenCalled(); + exitSpy.mockRestore(); + + expect(config.theGraphApiKey).toBe("my-api-key"); + }); + describe("Useful error messages", () => { - // Mock process.exit to prevent actual exit - const mockExit = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + let exitSpy: ReturnType; beforeEach(() => { vi.clearAllMocks(); + exitSpy = mockProcessExit(); }); afterEach(() => { - mockExit.mockClear(); + exitSpy.mockRestore(); }); - const TEST_ENV: EnsApiEnvironment = structuredClone(BASE_ENV); - it("logs error and exits when REFERRAL_PROGRAM_EDITIONS is not a valid URL", async () => { - await buildConfigFromEnvironment({ - ...TEST_ENV, - REFERRAL_PROGRAM_EDITIONS: "not-a-url", - }); + const testEnv = structuredClone(BASE_ENV); - expect(logger.error).toHaveBeenCalledWith( + await expect( + buildConfigFromEnvironment({ + ...testEnv, + REFERRAL_PROGRAM_EDITIONS: "not-a-url", + }), + ).rejects.toThrow("process.exit"); + + expect(logger.error).toHaveBeenCalledExactlyOnceWith( expect.stringContaining("REFERRAL_PROGRAM_EDITIONS is not a valid URL: not-a-url"), ); - expect(process.exit).toHaveBeenCalledWith(1); + expect(process.exit).toHaveBeenCalledExactlyOnceWith(1); }); it("logs error message when QuickNode RPC config was partially configured (missing endpoint name)", async () => { - await buildConfigFromEnvironment({ - ...TEST_ENV, - QUICKNODE_API_KEY: "my-api-key", - }); + const testEnv = structuredClone(BASE_ENV); + + await expect( + buildConfigFromEnvironment({ + ...testEnv, + QUICKNODE_API_KEY: "my-api-key", + }), + ).rejects.toThrow("process.exit"); - expect(logger.error).toHaveBeenCalledWith( + expect(logger.error).toHaveBeenCalledExactlyOnceWith( new Error( "Use of the QUICKNODE_API_KEY environment variable requires use of the QUICKNODE_ENDPOINT_NAME environment variable as well.", ), "Failed to build EnsApiConfig", ); - expect(process.exit).toHaveBeenCalledWith(1); + expect(process.exit).toHaveBeenCalledExactlyOnceWith(1); }); it("logs error message when QuickNode RPC config was partially configured (missing API key)", async () => { - await buildConfigFromEnvironment({ - ...TEST_ENV, - QUICKNODE_ENDPOINT_NAME: "my-endpoint-name", - }); + const testEnv = structuredClone(BASE_ENV); + + await expect( + buildConfigFromEnvironment({ + ...testEnv, + QUICKNODE_ENDPOINT_NAME: "my-endpoint-name", + }), + ).rejects.toThrow("process.exit"); - expect(logger.error).toHaveBeenCalledWith( + expect(logger.error).toHaveBeenCalledExactlyOnceWith( new Error( "Use of the QUICKNODE_ENDPOINT_NAME environment variable requires use of the QUICKNODE_API_KEY environment variable as well.", ), "Failed to build EnsApiConfig", ); - expect(process.exit).toHaveBeenCalledWith(1); + expect(process.exit).toHaveBeenCalledExactlyOnceWith(1); }); }); }); describe("buildEnsApiPublicConfig", () => { it("returns a valid ENSApi public config with correct structure", () => { - const mockConfig = { + const { ensIndexer: ensIndexerPublicConfig } = indexingMetadataContextInitialized.stackInfo; + const ensApiConfig = { port: ENSApi_DEFAULT_PORT, ensDbUrl: BASE_ENV.ENSDB_URL, - ensIndexerPublicConfig: ENSINDEXER_PUBLIC_CONFIG, - namespace: ENSINDEXER_PUBLIC_CONFIG.namespace, - ensIndexerSchemaName: ENSINDEXER_PUBLIC_CONFIG.ensIndexerSchemaName, + ensIndexerSchemaName: BASE_ENV.ENSINDEXER_SCHEMA_NAME, + ensIndexerPublicConfig, + namespace: ensIndexerPublicConfig.namespace, rpcConfigs: new Map([ [ 1, @@ -171,7 +187,7 @@ describe("buildEnsApiPublicConfig", () => { referralProgramEditionConfigSetUrl: undefined, }; - const result = buildEnsApiPublicConfig(mockConfig); + const result = buildEnsApiPublicConfig(ensApiConfig, ensIndexerPublicConfig); expect(result).toStrictEqual({ versionInfo: ensApiVersionInfo, @@ -179,50 +195,79 @@ describe("buildEnsApiPublicConfig", () => { canFallback: false, reason: "not-subgraph-compatible", }, - ensIndexerPublicConfig: ENSINDEXER_PUBLIC_CONFIG, + ensIndexerPublicConfig, }); }); it("preserves the complete ENSIndexer public config structure", () => { - const mockConfig = { + const { ensIndexer: ensIndexerPublicConfig } = indexingMetadataContextInitialized.stackInfo; + const ensApiConfig = { port: ENSApi_DEFAULT_PORT, ensDbUrl: BASE_ENV.ENSDB_URL, - ensIndexerPublicConfig: ENSINDEXER_PUBLIC_CONFIG, - namespace: ENSINDEXER_PUBLIC_CONFIG.namespace, - ensIndexerSchemaName: ENSINDEXER_PUBLIC_CONFIG.ensIndexerSchemaName, + ensIndexerSchemaName: BASE_ENV.ENSINDEXER_SCHEMA_NAME, + ensIndexerPublicConfig, + namespace: ensIndexerPublicConfig.namespace, rpcConfigs: new Map(), referralProgramEditionConfigSetUrl: undefined, }; - const result = buildEnsApiPublicConfig(mockConfig); + const result = buildEnsApiPublicConfig(ensApiConfig, ensIndexerPublicConfig); - // Verify that all ENSIndexer public config fields are preserved - expect(result.ensIndexerPublicConfig).toStrictEqual(ENSINDEXER_PUBLIC_CONFIG); + expect(result.ensIndexerPublicConfig).toStrictEqual(ensIndexerPublicConfig); }); it("includes the theGraphFallback and redacts api key", () => { - const mockConfig = { + const ensIndexerPublicConfig = { + ...indexingMetadataContextInitialized.stackInfo.ensIndexer, + plugins: ["subgraph"], + isSubgraphCompatible: true, + }; + + const ensApiConfig = { port: ENSApi_DEFAULT_PORT, ensDbUrl: BASE_ENV.ENSDB_URL, - ensIndexerPublicConfig: { - ...ENSINDEXER_PUBLIC_CONFIG, - plugins: ["subgraph"], - isSubgraphCompatible: true, - }, - namespace: ENSINDEXER_PUBLIC_CONFIG.namespace, - ensIndexerSchemaName: ENSINDEXER_PUBLIC_CONFIG.ensIndexerSchemaName, + ensIndexerSchemaName: BASE_ENV.ENSINDEXER_SCHEMA_NAME, + ensIndexerPublicConfig, + namespace: ensIndexerPublicConfig.namespace, rpcConfigs: new Map(), referralProgramEditionConfigSetUrl: undefined, theGraphApiKey: "secret-api-key", }; - const result = buildEnsApiPublicConfig(mockConfig); + const result = buildEnsApiPublicConfig(ensApiConfig, ensIndexerPublicConfig); expect(result.theGraphFallback.canFallback).toBe(true); // discriminate the type... if (!result.theGraphFallback.canFallback) throw new Error("never"); - // shouldn't have the secret-api-key in the url - expect(result.theGraphFallback.url).not.toMatch(/secret-api-key/gi); + expect(result.theGraphFallback.url).toBe( + "https://gateway.thegraph.com/api//subgraphs/id/5XqPmWe6gjyrJtFn9cLy237i4cWw2j9HcUJEXsP5qGtH", + ); + }); + + it("returns canFallback=false when no theGraphApiKey is provided even if subgraph compatible", () => { + const ensIndexerPublicConfig = { + ...indexingMetadataContextInitialized.stackInfo.ensIndexer, + plugins: ["subgraph"], + isSubgraphCompatible: true, + }; + + const ensApiConfig = { + port: ENSApi_DEFAULT_PORT, + ensDbUrl: BASE_ENV.ENSDB_URL, + ensIndexerSchemaName: BASE_ENV.ENSINDEXER_SCHEMA_NAME, + ensIndexerPublicConfig, + namespace: ensIndexerPublicConfig.namespace, + rpcConfigs: new Map(), + referralProgramEditionConfigSetUrl: undefined, + theGraphApiKey: undefined, + }; + + const result = buildEnsApiPublicConfig(ensApiConfig, ensIndexerPublicConfig); + + expect(result.theGraphFallback).toStrictEqual({ + canFallback: false, + reason: "no-api-key", + }); }); }); diff --git a/apps/ensapi/src/config/config.schema.ts b/apps/ensapi/src/config/config.schema.ts index 17576766f2..733bf0ec04 100644 --- a/apps/ensapi/src/config/config.schema.ts +++ b/apps/ensapi/src/config/config.schema.ts @@ -1,7 +1,11 @@ import pRetry from "p-retry"; import { prettifyError, ZodError, z } from "zod/v4"; -import type { EnsApiPublicConfig } from "@ensnode/ensnode-sdk"; +import { + type EnsApiPublicConfig, + type EnsIndexerPublicConfig, + IndexingMetadataContextStatusCodes, +} from "@ensnode/ensnode-sdk"; import { buildRpcConfigsFromEnv, canFallbackToTheGraph, @@ -70,13 +74,17 @@ export async function buildConfigFromEnvironment(env: EnsApiEnvironment): Promis // https://github.com/namehash/ensnode/issues/1806 const ensIndexerPublicConfig = await pRetry( async () => { - const config = await ensDbClient.getEnsIndexerPublicConfig(); + const indexingMetadataContext = await ensDbClient.getIndexingMetadataContext(); - if (!config) { - throw new Error("ENSIndexer Public Config not yet available in ENSDb."); + if ( + indexingMetadataContext.statusCode === IndexingMetadataContextStatusCodes.Uninitialized + ) { + throw new Error( + "EnsIndexerPublicConfig could not be fetched, the IndexingMetadataContext record has not been initialized in ENSDb yet.", + ); } - return config; + return indexingMetadataContext.stackInfo.ensIndexer; }, { retries: 13, // This allows for a total of over 1 hour of retries with the exponential backoff strategy @@ -121,17 +129,20 @@ export async function buildConfigFromEnvironment(env: EnsApiEnvironment): Promis * @param config - The validated EnsApiConfig object * @returns A complete ENSApiPublicConfig object */ -export function buildEnsApiPublicConfig(config: EnsApiConfig): EnsApiPublicConfig { +export function buildEnsApiPublicConfig( + ensApiConfig: EnsApiConfig, + ensIndexerPublicConfig: EnsIndexerPublicConfig, +): EnsApiPublicConfig { return { versionInfo: ensApiVersionInfo, theGraphFallback: canFallbackToTheGraph({ - namespace: config.namespace, + namespace: ensIndexerPublicConfig.namespace, // NOTE: very important here that we replace the actual server-side api key with a placeholder // so that it's not sent to clients as part of the `theGraphFallback.url`. The placeholder must // pass validation, of course, but the only validation necessary is that it is a string. - theGraphApiKey: config.theGraphApiKey ? "" : undefined, - isSubgraphCompatible: config.ensIndexerPublicConfig.isSubgraphCompatible, + theGraphApiKey: ensApiConfig.theGraphApiKey ? "" : undefined, + isSubgraphCompatible: ensIndexerPublicConfig.isSubgraphCompatible, }), - ensIndexerPublicConfig: config.ensIndexerPublicConfig, + ensIndexerPublicConfig, }; } diff --git a/apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts b/apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts index b4d3c9cb3e..463deec103 100644 --- a/apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts +++ b/apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts @@ -2,6 +2,7 @@ import { getUnixTime } from "date-fns"; import { Hono } from "hono"; import { + buildCrossChainIndexingStatusSnapshotOmnichain, createRealtimeIndexingStatusProjection, EnsIndexerIndexingStatusResponseCodes, type EnsIndexerIndexingStatusResponseError, @@ -10,45 +11,31 @@ import { serializeEnsIndexerPublicConfig, } from "@ensnode/ensnode-sdk"; -import { ensDbClient } from "@/lib/ensdb/singleton"; +import { indexingStatusBuilder } from "@/lib/indexing-status-builder/singleton"; import { logger } from "@/lib/logger"; +import { publicConfigBuilder } from "@/lib/public-config-builder/singleton"; const app = new Hono(); -// include ENSIndexer Public Config endpoint app.get("/config", async (c) => { - const publicConfig = await ensDbClient.getEnsIndexerPublicConfig(); - - // Invariant: the public config is guaranteed to be available in ENSDb after - // application startup. - if (typeof publicConfig === "undefined") { - throw new Error("Unreachable: ENSIndexer Public Config is not available in ENSDb"); - } + const ensIndexerPublicConfig = await publicConfigBuilder.getPublicConfig(); // respond with the serialized public config object - return c.json(serializeEnsIndexerPublicConfig(publicConfig)); + return c.json(serializeEnsIndexerPublicConfig(ensIndexerPublicConfig)); }); app.get("/indexing-status", async (c) => { try { - const crossChainSnapshot = await ensDbClient.getIndexingStatusSnapshot(); + const omnichainSnapshot = await indexingStatusBuilder.getOmnichainIndexingStatusSnapshot(); - // Invariant: the Indexing Status Snapshot is expected to be available in - // ENSDb shortly after application startup. There is a possibility that - // the snapshot is not yet available at the time of the request, - // i.e. when ENSDb has not yet been populated with the first snapshot. - // In this case, we treat the snapshot as unavailable and respond with - // an error response. - if (typeof crossChainSnapshot === "undefined") { - throw new Error("ENSDb does not contain an Indexing Status Snapshot"); - } - - const projectedAt = getUnixTime(new Date()); - const realtimeProjection = createRealtimeIndexingStatusProjection( - crossChainSnapshot, - projectedAt, + const now = getUnixTime(new Date()); + const crossChainSnapshot = buildCrossChainIndexingStatusSnapshotOmnichain( + omnichainSnapshot, + now, ); + const realtimeProjection = createRealtimeIndexingStatusProjection(crossChainSnapshot, now); + return c.json( serializeEnsIndexerIndexingStatusResponse({ responseCode: EnsIndexerIndexingStatusResponseCodes.Ok, diff --git a/apps/ensindexer/ponder/src/api/index.ts b/apps/ensindexer/ponder/src/api/index.ts index c00d161881..a714ce9e2b 100644 --- a/apps/ensindexer/ponder/src/api/index.ts +++ b/apps/ensindexer/ponder/src/api/index.ts @@ -5,27 +5,10 @@ import { cors } from "hono/cors"; import type { ErrorResponse } from "@ensnode/ensnode-sdk"; -import { migrateEnsNodeSchema } from "@/lib/ensdb/migrate-ensnode-schema"; -import { startEnsDbWriterWorker } from "@/lib/ensdb-writer-worker/singleton"; import { logger } from "@/lib/logger"; import ensNodeApi from "./handlers/ensnode-api"; -// Before starting the ENSDb Writer Worker, we need to ensure that -// the ENSNode Schema in ENSDb is up to date by running any pending migrations. -await migrateEnsNodeSchema().catch((error) => { - logger.error({ - msg: "Failed to initialize ENSNode metadata", - error, - module: "ponder-api", - }); - process.exitCode = 1; - throw error; -}); - -// The entry point for the ENSDb Writer Worker. -startEnsDbWriterWorker(); - const app = new Hono(); // set the X-ENSIndexer-Version header to the current version diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.mock.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.mock.ts index 50da45a6ff..30b9f4814b 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.mock.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.mock.ts @@ -2,85 +2,63 @@ import { vi } from "vitest"; import type { EnsDbWriter } from "@ensnode/ensdb-sdk"; import { + ChainIndexingStatusIds, type CrossChainIndexingStatusSnapshot, CrossChainIndexingStrategyIds, - ENSNamespaceIds, - type EnsIndexerPublicConfig, - type EnsIndexerVersionInfo, - type EnsRainbowPublicConfig, + type IndexingMetadataContextInitialized, + IndexingMetadataContextStatusCodes, OmnichainIndexingStatusIds, type OmnichainIndexingStatusSnapshot, - PluginName, + RangeTypeIds, } from "@ensnode/ensnode-sdk"; -import type { LocalPonderClient } from "@ensnode/ponder-sdk"; import { EnsDbWriterWorker } from "@/lib/ensdb-writer-worker/ensdb-writer-worker"; -import type { IndexingStatusBuilder } from "@/lib/indexing-status-builder"; -import type { PublicConfigBuilder } from "@/lib/public-config-builder"; +import type { IndexingMetadataContextBuilder } from "@/lib/indexing-metadata-context-builder/indexing-metadata-context-builder"; -// Test fixture for EnsRainbowPublicConfig -export const mockEnsRainbowPublicConfig: EnsRainbowPublicConfig = { - serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, - versionInfo: { - ensRainbow: "1.0.0", - }, -}; +// Test fixtures for IndexingMetadataContext objects -// Test fixture for EnsIndexerVersionInfo -export const mockVersionInfo: EnsIndexerVersionInfo = { - ponder: "0.9.0", - ensDb: "1.0.0", - ensIndexer: "1.0.0", - ensNormalize: "1.10.0", -}; - -// Test fixture for EnsIndexerPublicConfig -export const mockPublicConfig: EnsIndexerPublicConfig = { - ensIndexerSchemaName: "ensindexer_0", - clientLabelSet: { labelSetId: "subgraph", labelSetVersion: 0 }, - ensRainbowPublicConfig: mockEnsRainbowPublicConfig, - indexedChainIds: new Set([1, 8453]), - isSubgraphCompatible: true, - namespace: ENSNamespaceIds.Mainnet, - plugins: [PluginName.Subgraph], - versionInfo: mockVersionInfo, -}; - -// Helper to create mock objects with consistent typing -export function createMockEnsDbWriter( - overrides: Partial> = {}, -): EnsDbWriter { +export function createMockCrossChainSnapshot( + overrides: Partial = {}, +): CrossChainIndexingStatusSnapshot { return { - ...baseEnsDbWriter(), + strategy: CrossChainIndexingStrategyIds.Omnichain, + slowestChainIndexingCursor: 100, + snapshotTime: 200, + omnichainSnapshot: { + omnichainStatus: OmnichainIndexingStatusIds.Following, + omnichainIndexingCursor: 100, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Following, + latestIndexedBlock: { timestamp: 100, number: 100 }, + latestKnownBlock: { timestamp: 200, number: 200 }, + config: { + rangeType: RangeTypeIds.LeftBounded, + startBlock: { timestamp: 0, number: 0 }, + }, + }, + ], + ]), + }, ...overrides, - } as unknown as EnsDbWriter; -} - -export function baseEnsDbWriter() { - return { - getEnsDbVersion: vi.fn().mockResolvedValue(undefined), - getEnsIndexerPublicConfig: vi.fn().mockResolvedValue(undefined), - getIndexingStatusSnapshot: vi.fn().mockResolvedValue(undefined), - upsertEnsDbVersion: vi.fn().mockResolvedValue(undefined), - upsertEnsIndexerPublicConfig: vi.fn().mockResolvedValue(undefined), - upsertIndexingStatusSnapshot: vi.fn().mockResolvedValue(undefined), }; } -export function createMockPublicConfigBuilder( - resolvedConfig: EnsIndexerPublicConfig = mockPublicConfig, -): PublicConfigBuilder { - return { - getPublicConfig: vi.fn().mockResolvedValue(resolvedConfig), - } as unknown as PublicConfigBuilder; -} - -export function createMockIndexingStatusBuilder( - resolvedSnapshot: OmnichainIndexingStatusSnapshot = createMockOmnichainSnapshot(), -): IndexingStatusBuilder { +export function createMockIndexingMetadataContextInitialized( + overrides: Partial = {}, +): IndexingMetadataContextInitialized { return { - getOmnichainIndexingStatusSnapshot: vi.fn().mockResolvedValue(resolvedSnapshot), - } as unknown as IndexingStatusBuilder; + statusCode: IndexingMetadataContextStatusCodes.Initialized, + indexingStatus: createMockCrossChainSnapshot(), + stackInfo: { + ensDb: { versionInfo: { postgresql: "17.4" } }, + ensIndexer: {} as any, + ensRainbow: {} as any, + }, + ...overrides, + }; } export function createMockOmnichainSnapshot( @@ -94,48 +72,32 @@ export function createMockOmnichainSnapshot( }; } -export function createMockCrossChainSnapshot( - overrides: Partial = {}, -): CrossChainIndexingStatusSnapshot { +export function createMockEnsDbWriter( + overrides: Partial> = {}, +): EnsDbWriter { return { - strategy: CrossChainIndexingStrategyIds.Omnichain, - slowestChainIndexingCursor: 100, - snapshotTime: 200, - omnichainSnapshot: createMockOmnichainSnapshot(), + upsertIndexingMetadataContext: vi.fn().mockResolvedValue(undefined), ...overrides, - }; + } as unknown as EnsDbWriter; } -export function createMockLocalPonderClient( - overrides: { isInDevMode?: boolean } = {}, -): LocalPonderClient { - const isInDevMode = overrides.isInDevMode ?? false; - +export function createMockIndexingMetadataContextBuilder( + resolvedContext: IndexingMetadataContextInitialized = createMockIndexingMetadataContextInitialized(), +): IndexingMetadataContextBuilder { return { - isInDevMode, - } as unknown as LocalPonderClient; + getIndexingMetadataContext: vi.fn().mockResolvedValue(resolvedContext), + } as unknown as IndexingMetadataContextBuilder; } export function createMockEnsDbWriterWorker( overrides: { ensDbClient?: EnsDbWriter; - publicConfigBuilder?: PublicConfigBuilder; - indexingStatusBuilder?: IndexingStatusBuilder; - isInDevMode?: boolean; + indexingMetadataContextBuilder?: IndexingMetadataContextBuilder; } = {}, ) { const ensDbClient = overrides.ensDbClient ?? createMockEnsDbWriter(); - const publicConfigBuilder = overrides.publicConfigBuilder ?? createMockPublicConfigBuilder(); - const indexingStatusBuilder = - overrides.indexingStatusBuilder ?? createMockIndexingStatusBuilder(); - const localPonderClient = createMockLocalPonderClient({ - isInDevMode: overrides.isInDevMode ?? false, - }); + const indexingMetadataContextBuilder = + overrides.indexingMetadataContextBuilder ?? createMockIndexingMetadataContextBuilder(); - return new EnsDbWriterWorker( - ensDbClient, - publicConfigBuilder, - indexingStatusBuilder, - localPonderClient, - ); + return new EnsDbWriterWorker(ensDbClient, indexingMetadataContextBuilder); } diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts index ba0f0bee5b..00c5f72f44 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts @@ -1,40 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { - buildCrossChainIndexingStatusSnapshotOmnichain, - OmnichainIndexingStatusIds, - validateEnsIndexerPublicConfigCompatibility, -} from "@ensnode/ensnode-sdk"; - import "@/lib/__test__/mockLogger"; -import type { IndexingStatusBuilder } from "@/lib/indexing-status-builder/indexing-status-builder"; -import type { PublicConfigBuilder } from "@/lib/public-config-builder/public-config-builder"; - import { - createMockCrossChainSnapshot, createMockEnsDbWriter, createMockEnsDbWriterWorker, - createMockIndexingStatusBuilder, - createMockOmnichainSnapshot, - createMockPublicConfigBuilder, - mockPublicConfig, + createMockIndexingMetadataContextBuilder, + createMockIndexingMetadataContextInitialized, } from "./ensdb-writer-worker.mock"; -vi.mock("@ensnode/ensnode-sdk", async () => { - const actual = await vi.importActual("@ensnode/ensnode-sdk"); - - return { - ...actual, - validateEnsIndexerPublicConfigCompatibility: vi.fn(), - buildCrossChainIndexingStatusSnapshotOmnichain: vi.fn(), - }; -}); - -vi.mock("p-retry", () => ({ - default: vi.fn((fn) => fn()), -})); - describe("EnsDbWriterWorker", () => { beforeEach(() => { vi.useFakeTimers(); @@ -46,87 +20,26 @@ describe("EnsDbWriterWorker", () => { }); describe("run() - worker initialization", () => { - it("upserts version, config, and starts interval for indexing status snapshots", async () => { + it("starts the interval for updating indexing metadata context", async () => { // arrange - const omnichainSnapshot = createMockOmnichainSnapshot(); - const snapshot = createMockCrossChainSnapshot({ omnichainSnapshot }); - vi.mocked(buildCrossChainIndexingStatusSnapshotOmnichain).mockReturnValue(snapshot); + const context = createMockIndexingMetadataContextInitialized(); + const indexingMetadataContextBuilder = createMockIndexingMetadataContextBuilder(context); const ensDbClient = createMockEnsDbWriter(); const worker = createMockEnsDbWriterWorker({ ensDbClient, - indexingStatusBuilder: createMockIndexingStatusBuilder(omnichainSnapshot), + indexingMetadataContextBuilder, }); // act await worker.run(); - // assert - verify initial upserts happened - expect(ensDbClient.upsertEnsDbVersion).toHaveBeenCalledWith( - mockPublicConfig.versionInfo.ensDb, - ); - expect(ensDbClient.upsertEnsIndexerPublicConfig).toHaveBeenCalledWith(mockPublicConfig); - // advance time to trigger interval await vi.advanceTimersByTimeAsync(1000); - // assert - snapshot should be upserted - expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledWith(snapshot); - expect(buildCrossChainIndexingStatusSnapshotOmnichain).toHaveBeenCalledWith( - omnichainSnapshot, - expect.any(Number), - ); - - // cleanup - worker.stop(); - }); - - it("throws when stored config is incompatible", async () => { - // arrange - vi.mocked(validateEnsIndexerPublicConfigCompatibility).mockImplementation(() => { - throw new Error("incompatible"); - }); - - const ensDbClient = createMockEnsDbWriter({ - getEnsIndexerPublicConfig: vi.fn().mockResolvedValue(mockPublicConfig), - }); - const worker = createMockEnsDbWriterWorker({ - ensDbClient, - publicConfigBuilder: createMockPublicConfigBuilder(mockPublicConfig), - }); - - // act & assert - await expect(worker.run()).rejects.toThrow("incompatible"); - expect(ensDbClient.upsertEnsDbVersion).not.toHaveBeenCalled(); - }); - - it("skips config validation when in dev mode", async () => { - // arrange - vi.mocked(validateEnsIndexerPublicConfigCompatibility).mockImplementation(() => { - throw new Error("incompatible"); - }); - - const snapshot = createMockCrossChainSnapshot(); - vi.mocked(buildCrossChainIndexingStatusSnapshotOmnichain).mockReturnValue(snapshot); - - const ensDbClient = createMockEnsDbWriter({ - getEnsIndexerPublicConfig: vi.fn().mockResolvedValue(mockPublicConfig), - }); - const worker = createMockEnsDbWriterWorker({ - ensDbClient, - publicConfigBuilder: createMockPublicConfigBuilder(mockPublicConfig), - isInDevMode: true, - }); - - // act - should not throw even though configs are incompatible - await worker.run(); - - // assert - validation should not have been called - expect(validateEnsIndexerPublicConfigCompatibility).not.toHaveBeenCalled(); - expect(ensDbClient.upsertEnsDbVersion).toHaveBeenCalledWith( - mockPublicConfig.versionInfo.ensDb, - ); - expect(ensDbClient.upsertEnsIndexerPublicConfig).toHaveBeenCalledWith(mockPublicConfig); + // assert - worker delegates to indexingMetadataContextBuilder + expect(indexingMetadataContextBuilder.getIndexingMetadataContext).toHaveBeenCalled(); + expect(ensDbClient.upsertIndexingMetadataContext).toHaveBeenCalledWith(context); // cleanup worker.stop(); @@ -145,90 +58,20 @@ describe("EnsDbWriterWorker", () => { // cleanup worker.stop(); }); - - it("throws error when config fetch fails", async () => { - // arrange - const publicConfigBuilder = { - getPublicConfig: vi.fn().mockRejectedValue(new Error("Network failure")), - } as unknown as PublicConfigBuilder; - const ensDbClient = createMockEnsDbWriter(); - const worker = createMockEnsDbWriterWorker({ ensDbClient, publicConfigBuilder }); - - // act & assert - await expect(worker.run()).rejects.toThrow("Network failure"); - expect(publicConfigBuilder.getPublicConfig).toHaveBeenCalledTimes(1); - expect(ensDbClient.upsertEnsDbVersion).not.toHaveBeenCalled(); - }); - - it("throws error when stored config fetch fails", async () => { - // arrange - const ensDbClient = createMockEnsDbWriter({ - getEnsIndexerPublicConfig: vi.fn().mockRejectedValue(new Error("Database connection lost")), - }); - const worker = createMockEnsDbWriterWorker({ ensDbClient }); - - // act & assert - await expect(worker.run()).rejects.toThrow("Database connection lost"); - expect(ensDbClient.upsertEnsDbVersion).not.toHaveBeenCalled(); - }); - - it("fetches stored and in-memory configs concurrently", async () => { - // arrange - vi.mocked(validateEnsIndexerPublicConfigCompatibility).mockImplementation(() => {}); - - const ensDbClient = createMockEnsDbWriter({ - getEnsIndexerPublicConfig: vi.fn().mockResolvedValue(mockPublicConfig), - }); - const publicConfigBuilder = createMockPublicConfigBuilder(mockPublicConfig); - const worker = createMockEnsDbWriterWorker({ - ensDbClient, - publicConfigBuilder, - }); - - // act - await worker.run(); - - // assert - both should have been called (concurrent execution via Promise.all) - expect(ensDbClient.getEnsIndexerPublicConfig).toHaveBeenCalledTimes(1); - expect(publicConfigBuilder.getPublicConfig).toHaveBeenCalledTimes(1); - - // cleanup - worker.stop(); - }); - - it("calls pRetry for config fetch with retry logic", async () => { - // arrange - pRetry is mocked to call fn directly - const snapshot = createMockCrossChainSnapshot(); - vi.mocked(buildCrossChainIndexingStatusSnapshotOmnichain).mockReturnValue(snapshot); - - const ensDbClient = createMockEnsDbWriter(); - const publicConfigBuilder = createMockPublicConfigBuilder(); - const worker = createMockEnsDbWriterWorker({ ensDbClient, publicConfigBuilder }); - - // act - await worker.run(); - - // assert - config should be called once (pRetry is mocked) - expect(publicConfigBuilder.getPublicConfig).toHaveBeenCalledTimes(1); - expect(ensDbClient.upsertEnsIndexerPublicConfig).toHaveBeenCalledWith(mockPublicConfig); - - // cleanup - worker.stop(); - }); }); describe("stop() - worker termination", () => { it("stops the interval when stop() is called", async () => { // arrange - const upsertIndexingStatusSnapshot = vi.fn().mockResolvedValue(undefined); - const ensDbClient = createMockEnsDbWriter({ upsertIndexingStatusSnapshot }); + const upsertIndexingMetadataContext = vi.fn().mockResolvedValue(undefined); + const ensDbClient = createMockEnsDbWriter({ upsertIndexingMetadataContext }); const worker = createMockEnsDbWriterWorker({ ensDbClient }); // act await worker.run(); await vi.advanceTimersByTimeAsync(1000); - const callCountBeforeStop = upsertIndexingStatusSnapshot.mock.calls.length; + const callCountBeforeStop = upsertIndexingMetadataContext.mock.calls.length; worker.stop(); @@ -236,7 +79,7 @@ describe("EnsDbWriterWorker", () => { await vi.advanceTimersByTimeAsync(2000); // assert - no more calls after stop - expect(upsertIndexingStatusSnapshot).toHaveBeenCalledTimes(callCountBeforeStop); + expect(upsertIndexingMetadataContext).toHaveBeenCalledTimes(callCountBeforeStop); }); }); @@ -262,103 +105,130 @@ describe("EnsDbWriterWorker", () => { }); }); - describe("interval behavior - snapshot upserts", () => { - it("continues upserting after snapshot validation errors", async () => { + describe("interval behavior - updateIndexingMetadataContext", () => { + it("calls getIndexingMetadataContext and upserts on each tick", async () => { // arrange - const unstartedSnapshot = createMockOmnichainSnapshot({ - omnichainStatus: OmnichainIndexingStatusIds.Unstarted, - }); - const validSnapshot = createMockOmnichainSnapshot({ - omnichainIndexingCursor: 200, - }); - const crossChainSnapshot = createMockCrossChainSnapshot({ - slowestChainIndexingCursor: 200, - snapshotTime: 300, - omnichainSnapshot: validSnapshot, - }); + const context1 = createMockIndexingMetadataContextInitialized(); + const context2 = createMockIndexingMetadataContextInitialized(); - vi.mocked(buildCrossChainIndexingStatusSnapshotOmnichain).mockReturnValue(crossChainSnapshot); + const indexingMetadataContextBuilder = createMockIndexingMetadataContextBuilder(context1); + (indexingMetadataContextBuilder.getIndexingMetadataContext as any) + .mockResolvedValueOnce(context1) + .mockResolvedValueOnce(context2); const ensDbClient = createMockEnsDbWriter(); - const indexingStatusBuilder = { - getOmnichainIndexingStatusSnapshot: vi - .fn() - .mockResolvedValueOnce(unstartedSnapshot) - .mockResolvedValueOnce(validSnapshot), - } as unknown as IndexingStatusBuilder; - const worker = createMockEnsDbWriterWorker({ ensDbClient, indexingStatusBuilder }); + const worker = createMockEnsDbWriterWorker({ + ensDbClient, + indexingMetadataContextBuilder, + }); - // act - run returns immediately + // act await worker.run(); - // first interval tick - should error but not throw + // first tick await vi.advanceTimersByTimeAsync(1000); + expect(indexingMetadataContextBuilder.getIndexingMetadataContext).toHaveBeenCalledTimes(1); + expect(ensDbClient.upsertIndexingMetadataContext).toHaveBeenCalledWith(context1); - // second interval tick - should succeed + // second tick await vi.advanceTimersByTimeAsync(1000); - - // assert - expect(indexingStatusBuilder.getOmnichainIndexingStatusSnapshot).toHaveBeenCalledTimes(2); - expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledTimes(1); - expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledWith(crossChainSnapshot); + expect(indexingMetadataContextBuilder.getIndexingMetadataContext).toHaveBeenCalledTimes(2); + expect(ensDbClient.upsertIndexingMetadataContext).toHaveBeenCalledWith(context2); // cleanup worker.stop(); }); - it("recovers from errors and continues upserting snapshots", async () => { + it("recovers from getIndexingMetadataContext errors between ticks", async () => { // arrange - const snapshot1 = createMockOmnichainSnapshot({ omnichainIndexingCursor: 100 }); - const snapshot2 = createMockOmnichainSnapshot({ omnichainIndexingCursor: 200 }); + const context = createMockIndexingMetadataContextInitialized(); + const indexingMetadataContextBuilder = createMockIndexingMetadataContextBuilder(context); + (indexingMetadataContextBuilder.getIndexingMetadataContext as any) + .mockResolvedValueOnce(context) + .mockRejectedValueOnce(new Error("Builder error")) + .mockResolvedValueOnce(context); - const crossChainSnapshot1 = createMockCrossChainSnapshot({ - slowestChainIndexingCursor: 100, - snapshotTime: 1000, - omnichainSnapshot: snapshot1, - }); - const crossChainSnapshot2 = createMockCrossChainSnapshot({ - slowestChainIndexingCursor: 200, - snapshotTime: 2000, - omnichainSnapshot: snapshot2, + const ensDbClient = createMockEnsDbWriter(); + const worker = createMockEnsDbWriterWorker({ + ensDbClient, + indexingMetadataContextBuilder, }); - vi.mocked(buildCrossChainIndexingStatusSnapshotOmnichain) - .mockReturnValueOnce(crossChainSnapshot1) - .mockReturnValueOnce(crossChainSnapshot2) - .mockReturnValueOnce(crossChainSnapshot2); + // act + await worker.run(); + + // first tick - succeeds + await vi.advanceTimersByTimeAsync(1000); + expect(ensDbClient.upsertIndexingMetadataContext).toHaveBeenCalledTimes(1); + + // second tick - builder error, swallowed, no upsert + await vi.advanceTimersByTimeAsync(1000); + expect(ensDbClient.upsertIndexingMetadataContext).toHaveBeenCalledTimes(1); // no new upsert + + // third tick - succeeds again + await vi.advanceTimersByTimeAsync(1000); + expect(ensDbClient.upsertIndexingMetadataContext).toHaveBeenCalledTimes(2); + + // cleanup + worker.stop(); + }); + + it("recovers from upsertIndexingMetadataContext errors between ticks", async () => { + // arrange + const context = createMockIndexingMetadataContextInitialized(); + const indexingMetadataContextBuilder = createMockIndexingMetadataContextBuilder(context); const ensDbClient = createMockEnsDbWriter({ - upsertIndexingStatusSnapshot: vi + upsertIndexingMetadataContext: vi .fn() .mockResolvedValueOnce(undefined) .mockRejectedValueOnce(new Error("DB error")) .mockResolvedValueOnce(undefined), }); - const indexingStatusBuilder = { - getOmnichainIndexingStatusSnapshot: vi - .fn() - .mockResolvedValueOnce(snapshot1) - .mockResolvedValueOnce(snapshot2) - .mockResolvedValueOnce(snapshot2), - } as unknown as IndexingStatusBuilder; - const worker = createMockEnsDbWriterWorker({ ensDbClient, indexingStatusBuilder }); + + const worker = createMockEnsDbWriterWorker({ + ensDbClient, + indexingMetadataContextBuilder, + }); // act await worker.run(); // first tick - succeeds await vi.advanceTimersByTimeAsync(1000); - expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledWith(crossChainSnapshot1); + expect(ensDbClient.upsertIndexingMetadataContext).toHaveBeenCalledTimes(1); - // second tick - fails with DB error, but continues + // second tick - DB error, swallowed, upsert was called but rejected await vi.advanceTimersByTimeAsync(1000); - expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenLastCalledWith( - crossChainSnapshot2, - ); + expect(ensDbClient.upsertIndexingMetadataContext).toHaveBeenCalledTimes(2); // third tick - succeeds again await vi.advanceTimersByTimeAsync(1000); - expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledTimes(3); + expect(ensDbClient.upsertIndexingMetadataContext).toHaveBeenCalledTimes(3); + + // cleanup + worker.stop(); + }); + + it("does not stop worker or set exitCode on error", async () => { + // arrange + const indexingMetadataContextBuilder = createMockIndexingMetadataContextBuilder(); + (indexingMetadataContextBuilder.getIndexingMetadataContext as any).mockRejectedValue( + new Error("Fatal error"), + ); + + const worker = createMockEnsDbWriterWorker({ indexingMetadataContextBuilder }); + + // reset exitCode before test + process.exitCode = undefined; + + // act + await worker.run(); + await vi.advanceTimersByTimeAsync(1000); + + // assert - error is swallowed, worker keeps running, no exitCode set + expect(worker.isRunning).toBe(true); + expect(process.exitCode).toBeUndefined(); // cleanup worker.stop(); diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts index 38aa6d12de..c28d86bdab 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts @@ -1,95 +1,59 @@ -import { getUnixTime, secondsToMilliseconds } from "date-fns"; +import { secondsToMilliseconds } from "date-fns"; import type { Duration } from "enssdk"; -import pRetry from "p-retry"; import type { EnsDbWriter } from "@ensnode/ensdb-sdk"; -import { - buildCrossChainIndexingStatusSnapshotOmnichain, - type CrossChainIndexingStatusSnapshot, - type EnsIndexerPublicConfig, - OmnichainIndexingStatusIds, - type OmnichainIndexingStatusSnapshot, - validateEnsIndexerPublicConfigCompatibility, -} from "@ensnode/ensnode-sdk"; -import type { LocalPonderClient } from "@ensnode/ponder-sdk"; -import type { IndexingStatusBuilder } from "@/lib/indexing-status-builder/indexing-status-builder"; +import type { IndexingMetadataContextBuilder } from "@/lib/indexing-metadata-context-builder/indexing-metadata-context-builder"; import { logger } from "@/lib/logger"; -import type { PublicConfigBuilder } from "@/lib/public-config-builder/public-config-builder"; /** * Interval in seconds between two consecutive attempts to upsert - * the Indexing Status Snapshot record into ENSDb. + * the IndexingMetadataContext record into ENSDb. */ -const INDEXING_STATUS_RECORD_UPDATE_INTERVAL: Duration = 1; +const INDEXING_METADATA_CONTEXT_RECORD_UPDATE_INTERVAL: Duration = 1; /** * ENSDb Writer Worker * - * A worker responsible for writing ENSIndexer-related metadata into ENSDb, including: - * - ENSDb version - * - ENSIndexer Public Config - * - ENSIndexer Indexing Status Snapshots + * A worker responsible for writing the current {@link CrossChainIndexingStatusSnapshot} into + * the {@link IndexingMetadataContext} record in ENSDb. */ export class EnsDbWriterWorker { /** - * Interval for recurring upserts of Indexing Status Snapshots into ENSDb. + * Interval for recurring updates of IndexingMetadataContext record in ENSDb. */ - private indexingStatusInterval: ReturnType | null = null; + private indexingMetadataContextUpdateInterval: ReturnType | null = null; /** - * ENSDb Client instance used by the worker to interact with ENSDb. + * {@link EnsDbWriter} instance used by the worker to interact with ENSDb instance. */ private ensDbClient: EnsDbWriter; /** - * Indexing Status Builder instance used by the worker to read ENSIndexer Indexing Status. + * {@link IndexingMetadataContextBuilder} instance used by the worker to read {@link IndexingMetadataContext}. */ - private indexingStatusBuilder: IndexingStatusBuilder; - - /** - * ENSIndexer Public Config Builder instance used by the worker to read ENSIndexer Public Config. - */ - private publicConfigBuilder: PublicConfigBuilder; - - /** - * Local Ponder Client instance - * - * Used to get local Ponder app command. - */ - private localPonderClient: LocalPonderClient; + private indexingMetadataContextBuilder: IndexingMetadataContextBuilder; /** * @param ensDbClient ENSDb Writer instance used by the worker to interact with ENSDb. - * @param publicConfigBuilder ENSIndexer Public Config Builder instance used by the worker to read ENSIndexer Public Config. - * @param indexingStatusBuilder Indexing Status Builder instance used by the worker to read ENSIndexer Indexing Status. - * @param localPonderClient Local Ponder Client instance, used to get local Ponder app command. + * @param indexingMetadataContextBuilder {@link IndexingMetadataContextBuilder} instance used by + * the worker to read {@link IndexingMetadataContext}. */ constructor( ensDbClient: EnsDbWriter, - publicConfigBuilder: PublicConfigBuilder, - indexingStatusBuilder: IndexingStatusBuilder, - localPonderClient: LocalPonderClient, + indexingMetadataContextBuilder: IndexingMetadataContextBuilder, ) { this.ensDbClient = ensDbClient; - this.publicConfigBuilder = publicConfigBuilder; - this.indexingStatusBuilder = indexingStatusBuilder; - this.localPonderClient = localPonderClient; + this.indexingMetadataContextBuilder = indexingMetadataContextBuilder; } /** * Run the ENSDb Writer Worker * - * The worker performs the following tasks: - * 1) A single attempt to upsert ENSDb version into ENSDb. - * 2) A single attempt to upsert serialized representation of - * {@link EnsIndexerPublicConfig} into ENSDb. - * 3) A recurring attempt to upsert serialized representation of - * {@link CrossChainIndexingStatusSnapshot} into ENSDb. + * The worker performs a recurring upsert of + * the {@link IndexingMetadataContext} record into ENSDb. * - * @throws Error if the worker is already running, or - * if the in-memory ENSIndexer Public Config could not be fetched, or - * if the in-memory ENSIndexer Public Config is incompatible with the stored config in ENSDb. + * @throws Error if the worker is already running. */ public async run(): Promise { // Do not allow multiple concurrent runs of the worker @@ -97,33 +61,17 @@ export class EnsDbWriterWorker { throw new Error("EnsDbWriterWorker is already running"); } - // Fetch data required for task 1 and task 2. - const inMemoryConfig = await this.getValidatedEnsIndexerPublicConfig(); - - // Task 1: upsert ENSDb version into ENSDb. - logger.debug({ msg: "Upserting ENSDb version", module: "EnsDbWriterWorker" }); - await this.ensDbClient.upsertEnsDbVersion(inMemoryConfig.versionInfo.ensDb); - logger.info({ - msg: "Upserted ENSDb version", - ensDbVersion: inMemoryConfig.versionInfo.ensDb, - module: "EnsDbWriterWorker", - }); - - // Task 2: upsert of EnsIndexerPublicConfig into ENSDb. - logger.debug({ - msg: "Upserting ENSIndexer public config", - module: "EnsDbWriterWorker", - }); - await this.ensDbClient.upsertEnsIndexerPublicConfig(inMemoryConfig); - logger.info({ - msg: "Upserted ENSIndexer public config", - module: "EnsDbWriterWorker", - }); - - // Task 3: recurring upsert of Indexing Status Snapshot into ENSDb. - this.indexingStatusInterval = setInterval( - () => this.upsertIndexingStatusSnapshot(), - secondsToMilliseconds(INDEXING_STATUS_RECORD_UPDATE_INTERVAL), + // Recurring update of the IndexingMetadataContext record in ENSDb. + this.indexingMetadataContextUpdateInterval = setInterval( + () => + this.updateIndexingMetadataContext().catch((error) => { + logger.error({ + msg: "Failed to update indexing metadata context record in ENSDb", + module: "EnsDbWriterWorker", + error, + }); + }), + secondsToMilliseconds(INDEXING_METADATA_CONTEXT_RECORD_UPDATE_INTERVAL), ); } @@ -131,7 +79,7 @@ export class EnsDbWriterWorker { * Indicates whether the ENSDb Writer Worker is currently running. */ get isRunning(): boolean { - return this.indexingStatusInterval !== null; + return this.indexingMetadataContextUpdateInterval !== null; } /** @@ -140,162 +88,23 @@ export class EnsDbWriterWorker { * Stops all recurring tasks in the worker. */ public stop(): void { - if (this.indexingStatusInterval) { - clearInterval(this.indexingStatusInterval); - this.indexingStatusInterval = null; + if (this.indexingMetadataContextUpdateInterval) { + clearInterval(this.indexingMetadataContextUpdateInterval); + this.indexingMetadataContextUpdateInterval = null; } } /** - * Get validated ENSIndexer Public Config object for the ENSDb Writer Worker. + * Update the current IndexingMetadataContext into ENSDb. * - * The function retrieves the ENSIndexer Public Config object from both: - * - stored config in ENSDb, if available, and - * - in-memory config from ENSIndexer Client. + * This method is called by the scheduler at regular intervals from {@link run}. * - * If a stored config exists **and** the local Ponder app is **not** in dev - * mode, the in-memory config is validated for compatibility against the - * stored one. Validation is skipped if the local Ponder app is in dev mode, - * allowing to override the stored config in ENSDb with the current in-memory - * config, without having to keep them compatible. - * - * @returns The in-memory config when validation passes or no stored config - * exists. - * @throws Error if either fetch fails, or if the in-memory config is - * incompatible with the stored config. + * @throws Error if the update operation fails. */ - private async getValidatedEnsIndexerPublicConfig(): Promise { - /** - * Fetch the in-memory config with retries, to handle potential transient errors - * in the ENSIndexer Public Config Builder (e.g. due to network issues). - * If the fetch fails after the defined number of retries, the error - * will be thrown and the worker will not start, as the ENSIndexer Public Config - * is a critical dependency for the worker's tasks. - */ - const configFetchRetries = 3; - - logger.debug({ - msg: "Fetching ENSIndexer public config", - retries: configFetchRetries, - module: "EnsDbWriterWorker", - }); - - const inMemoryConfigPromise = pRetry(() => this.publicConfigBuilder.getPublicConfig(), { - retries: configFetchRetries, - onFailedAttempt: ({ attemptNumber, retriesLeft }) => { - logger.warn({ - msg: "Config fetch attempt failed", - attempt: attemptNumber, - retriesLeft, - module: "EnsDbWriterWorker", - }); - }, - }); - - let storedConfig: EnsIndexerPublicConfig | undefined; - let inMemoryConfig: EnsIndexerPublicConfig; - - try { - [storedConfig, inMemoryConfig] = await Promise.all([ - this.ensDbClient.getEnsIndexerPublicConfig(), - inMemoryConfigPromise, - ]); - logger.info({ - msg: "Fetched ENSIndexer public config", - module: "EnsDbWriterWorker", - config: inMemoryConfig, - }); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : "Unknown error"; - - logger.error({ - msg: "Failed to fetch ENSIndexer public config", - error, - module: "EnsDbWriterWorker", - }); - - // Throw the error to terminate the ENSIndexer process due to failed fetch of critical dependency - throw new Error(errorMessage, { - cause: error, - }); - } - - // Validate in-memory config object compatibility with the stored one, - // if the stored one is available. - // The validation is skipped if the local Ponder app is running in dev mode. - // This is to improve the development experience during ENSIndexer - // development, by allowing to override the stored config in ENSDb with - // the current in-memory config, without having to keep them compatible. - if (storedConfig && !this.localPonderClient.isInDevMode) { - try { - validateEnsIndexerPublicConfigCompatibility(storedConfig, inMemoryConfig); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : "Unknown error"; - - logger.error({ - msg: "In-memory config incompatible with stored config", - error, - module: "EnsDbWriterWorker", - }); - - // Throw the error to terminate the ENSIndexer process due to - // found config incompatibility - throw new Error(errorMessage, { - cause: error, - }); - } - } - - return inMemoryConfig; - } - - /** - * Upsert the current Indexing Status Snapshot into ENSDb. - * - * This method is called by the scheduler at regular intervals. - * Errors are logged but not thrown, to keep the worker running. - */ - private async upsertIndexingStatusSnapshot(): Promise { - try { - // get system timestamp for the current iteration - const snapshotTime = getUnixTime(new Date()); - - const omnichainSnapshot = await this.getValidatedIndexingStatusSnapshot(); - - const crossChainSnapshot = buildCrossChainIndexingStatusSnapshotOmnichain( - omnichainSnapshot, - snapshotTime, - ); - - await this.ensDbClient.upsertIndexingStatusSnapshot(crossChainSnapshot); - } catch (error) { - logger.error({ - msg: "Failed to upsert indexing status snapshot", - error, - module: "EnsDbWriterWorker", - }); - // Do not throw the error, as failure to retrieve the Indexing Status - // should not cause the ENSDb Writer Worker to stop functioning. - } - } - - /** - * Get validated Omnichain Indexing Status Snapshot - * - * @returns Validated Omnichain Indexing Status Snapshot. - * @throws Error if the Omnichain Indexing Status is not in expected status yet. - */ - private async getValidatedIndexingStatusSnapshot(): Promise { - const omnichainSnapshot = await this.indexingStatusBuilder.getOmnichainIndexingStatusSnapshot(); - - // It only makes sense to write Indexing Status Snapshots into ENSDb once - // the indexing process has started, as before that there is no meaningful - // status to record. - // Invariant: the Omnichain Status must indicate that indexing has started already. - if (omnichainSnapshot.omnichainStatus === OmnichainIndexingStatusIds.Unstarted) { - throw new Error("Omnichain Status must not be 'Unstarted'."); - } + private async updateIndexingMetadataContext(): Promise { + const indexingMetadataContext = + await this.indexingMetadataContextBuilder.getIndexingMetadataContext(); - return omnichainSnapshot; + await this.ensDbClient.upsertIndexingMetadataContext(indexingMetadataContext); } } diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts index 22fd6a5e9b..66e10d9039 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts @@ -1,8 +1,6 @@ import { ensDbClient } from "@/lib/ensdb/singleton"; -import { indexingStatusBuilder } from "@/lib/indexing-status-builder/singleton"; -import { localPonderClient } from "@/lib/local-ponder-client"; +import { indexingMetadataContextBuilder } from "@/lib/indexing-metadata-context-builder/singleton"; import { logger } from "@/lib/logger"; -import { publicConfigBuilder } from "@/lib/public-config-builder/singleton"; import { EnsDbWriterWorker } from "./ensdb-writer-worker"; @@ -22,12 +20,7 @@ export function startEnsDbWriterWorker() { throw new Error("EnsDbWriterWorker has already been initialized"); } - ensDbWriterWorker = new EnsDbWriterWorker( - ensDbClient, - publicConfigBuilder, - indexingStatusBuilder, - localPonderClient, - ); + ensDbWriterWorker = new EnsDbWriterWorker(ensDbClient, indexingMetadataContextBuilder); ensDbWriterWorker .run() diff --git a/apps/ensindexer/src/lib/ensrainbow/singleton.ts b/apps/ensindexer/src/lib/ensrainbow/singleton.ts index 331d62e6a9..c6e11a2100 100644 --- a/apps/ensindexer/src/lib/ensrainbow/singleton.ts +++ b/apps/ensindexer/src/lib/ensrainbow/singleton.ts @@ -24,6 +24,69 @@ export const ensRainbowClient = new EnsRainbowApiClient({ clientLabelSet, }); +/** + * Cached promise for waiting for ENSRainbow to be healthy. + * + * This ensures that multiple concurrent calls to + * {@link waitForEnsRainbowToBeHealthy} will share the same underlying promise + * in order to use the same retry sequence. + */ +let waitForEnsRainbowToBeHealthyPromise: Promise | undefined; + +/** + * Wait for ENSRainbow to be healthy + * + * Blocks execution until the ENSRainbow instance is healthy. That is, + * the ENSRainbow instance is responsive and able to serve basic requests successfully. + * + * We need to wait for ENSRainbow to be healthy before attempting to fetch + * the {@link EnsRainbowPublicConfig} from ENSRainbow. + * + * @throws When ENSRainbow fails to become healthy after all configured retry attempts. + * This error will trigger termination of the ENSIndexer process. + */ +export function waitForEnsRainbowToBeHealthy(): Promise { + if (waitForEnsRainbowToBeHealthyPromise) { + return waitForEnsRainbowToBeHealthyPromise; + } + + logger.info({ + msg: `Waiting for ENSRainbow instance to be healthy`, + ensRainbowInstance: ensRainbowUrl.href, + }); + + waitForEnsRainbowToBeHealthyPromise = pRetry(async () => ensRainbowClient.health(), { + retries: 3, + onFailedAttempt: ({ attemptNumber, retriesLeft }) => { + logger.warn({ + msg: `ENSRainbow health check failed`, + attempt: attemptNumber, + retriesLeft, + ensRainbowInstance: ensRainbowUrl.href, + advice: `This might be a transient issue after ENSNode deployment. If this persists, it might indicate an issue with the ENSRainbow instance or connectivity to it.`, + }); + }, + }) + .then(() => { + logger.info({ + msg: `ENSRainbow instance is healthy`, + ensRainbowInstance: ensRainbowUrl.href, + }); + }) + .catch((error) => { + logger.error({ + msg: `ENSRainbow health check failed after multiple attempts`, + error, + ensRainbowInstance: ensRainbowUrl.href, + }); + + // Throw the error to terminate the ENSIndexer process due to the failed health check of a critical dependency + throw error; + }); + + return waitForEnsRainbowToBeHealthyPromise; +} + /** * Cached promise for waiting for ENSRainbow to be ready. * @@ -56,21 +119,30 @@ export function waitForEnsRainbowToBeReady(): Promise { ensRainbowInstance: ensRainbowUrl.href, }); - waitForEnsRainbowToBeReadyPromise = pRetry(async () => ensRainbowClient.health(), { - retries: 60, // This allows for a total of over 1 hour of retries with 1 minute between attempts. - minTimeout: secondsToMilliseconds(60), - maxTimeout: secondsToMilliseconds(60), - onFailedAttempt: ({ error, attemptNumber, retriesLeft }) => { - logger.warn({ - msg: `ENSRainbow health check failed`, - attempt: attemptNumber, - retriesLeft, - error: retriesLeft === 0 ? error : undefined, - ensRainbowInstance: ensRainbowUrl.href, - advice: `This might be due to ENSRainbow having a cold start, which can take 30+ minutes.`, - }); + waitForEnsRainbowToBeReadyPromise = pRetry( + // TODO: replace this count check with an explicit `ready()` method in ENSRainbow Client. + async () => { + const { count } = await ensRainbowClient.count(); + + if (count === 0) { + throw new Error("ENSRainbow instance is not ready yet."); + } }, - }) + { + retries: 60, // This allows for a total of over 1 hour of retries with 1 minute between attempts. + minTimeout: secondsToMilliseconds(60), + maxTimeout: secondsToMilliseconds(60), + onFailedAttempt: ({ attemptNumber, retriesLeft }) => { + logger.warn({ + msg: `ENSRainbow readiness check failed`, + attempt: attemptNumber, + retriesLeft, + ensRainbowInstance: ensRainbowUrl.href, + advice: `This might be due to ENSRainbow having a cold start, which can take 30+ minutes.`, + }); + }, + }, + ) .then(() => { logger.info({ msg: `ENSRainbow instance is ready`, @@ -78,18 +150,14 @@ export function waitForEnsRainbowToBeReady(): Promise { }); }) .catch((error) => { - const errorMessage = error instanceof Error ? error.message : "Unknown error"; - logger.error({ - msg: `ENSRainbow health check failed after multiple attempts`, + msg: `ENSRainbow readiness check failed after multiple attempts`, error, ensRainbowInstance: ensRainbowUrl.href, }); // Throw the error to terminate the ENSIndexer process due to the failed health check of a critical dependency - throw new Error(errorMessage, { - cause: error instanceof Error ? error : undefined, - }); + throw error; }); return waitForEnsRainbowToBeReadyPromise; diff --git a/apps/ensindexer/src/lib/indexing-engines/init-indexing-onchain-events.ts b/apps/ensindexer/src/lib/indexing-engines/init-indexing-onchain-events.ts new file mode 100644 index 0000000000..cf8c22fe44 --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-engines/init-indexing-onchain-events.ts @@ -0,0 +1,106 @@ +/** + * This module defines the initialization logic to be executed by + * the ENSIndexer instance before it starts executing any "onchain" + * event handlers. + */ + +import { migrateEnsNodeSchema } from "@/lib/ensdb/migrate-ensnode-schema"; +import { ensDbClient } from "@/lib/ensdb/singleton"; +import { startEnsDbWriterWorker } from "@/lib/ensdb-writer-worker/singleton"; +import { + waitForEnsRainbowToBeHealthy, + waitForEnsRainbowToBeReady, +} from "@/lib/ensrainbow/singleton"; +import { indexingMetadataContextBuilder } from "@/lib/indexing-metadata-context-builder/singleton"; +import { logger } from "@/lib/logger"; + +async function upsertIndexingMetadataContextRecord(): Promise { + const indexingMetadataContext = await indexingMetadataContextBuilder.getIndexingMetadataContext(); + + logger.info({ + msg: `Upserting Indexing Metadata Context Initialized`, + }); + logger.debug({ + msg: `Indexing Metadata Context`, + indexingStatus: indexingMetadataContext.indexingStatus, + stackInfo: indexingMetadataContext.stackInfo, + }); + + await ensDbClient.upsertIndexingMetadataContext(indexingMetadataContext); + + logger.info({ + msg: `Successfully upserted Indexing Metadata Context Initialized`, + }); +} + +/** + * Initialize indexing of "onchain" events + * + * This function is guaranteed to be called exactly once by + * `eventHandlerPreconditions` before executing any "onchain" event handlers, + * and is used to initialize the ENSNode Schema. + * + * Each time the ENSIndexer instance starts, the logic in this function will be + * executed. Therefore, all logic must be idempotent and concurrency-safe, + * to prevent any issues during the startup of the ENSIndexer instance. + * For example, multiple ENSIndexer instances might be started at the same time, + * and they might all execute the logic in this function concurrently, + * so we need to make sure that this does not cause any unexpected side effects. + */ +export async function initIndexingOnchainEvents(): Promise { + try { + // Ensure ENSDb instance is healthy before trying to run any queries against it. + const isEnsDbHealthy = await ensDbClient.isHealthy(); + + // Invariant: ENSDb instance must be healthy by now. + if (!isEnsDbHealthy) { + throw new Error("ENSDb instance must be healthy"); + } + + // Ensure the ENSNode Schema in ENSDb is up to date by running any pending migrations. + await migrateEnsNodeSchema(); + + // Before calling `ensRainbowClient.config()`, we want to make sure that + // the ENSRainbow instance is healthy and ready to serve requests. + // This is a quick check, as we expect the ENSRainbow instance to be healthy + // by the time ENSIndexer instance executes `initIndexingOnchainEvents`. + await waitForEnsRainbowToBeHealthy(); + + // Upsert the Indexing Metadata Context record into ENSDb + await upsertIndexingMetadataContextRecord(); + + // Invariant: at this point, the ENSDb instance must be considered ready. + // This is a defensive check, highly unlikely to ever fail, since we just + // have successfully executed database migrations for the ENSNode Schema + // and upserted the IndexingMetadataContext record into ENSDb. However, + // if any database migration silently failed without throwing an error, + // or if the upsert operation for IndexingMetadataContext record was + // not completed as expected, the ENSDb instance might not be ready, and + // we want to catch this issue before we start processing onchain events. + const isEnsDbReady = await ensDbClient.isReady(); + if (!isEnsDbReady) { + throw new Error("ENSDb instance must be ready before onchain events can be indexed."); + } + + // Before starting to process onchain events, we want to make sure that + // ENSRainbow is ready to serve the "heal" requests. + await waitForEnsRainbowToBeReady(); + + // TODO: start Indexing Status Sync worker + // It will be responsible for keeping the indexing status stored within Indexing Metadata Context record in ENSDb up to date + // await indexingStatusSyncWorker.start(); + startEnsDbWriterWorker(); + } catch (error) { + // If any error happens during the initialization of indexing of onchain events, + // we want to log the error and exit the process with a non-zero exit code, + // since this is a critical failure that prevents the ENSIndexer instance from functioning properly. + logger.error({ + msg: "Failed to initialize the onchain events indexing", + module: "init-indexing-onchain-events", + error, + }); + + process.exitCode = 1; + throw error; + } +} diff --git a/apps/ensindexer/src/lib/indexing-engines/ponder.test.ts b/apps/ensindexer/src/lib/indexing-engines/ponder.test.ts index f506f918a5..25cb9ec44c 100644 --- a/apps/ensindexer/src/lib/indexing-engines/ponder.test.ts +++ b/apps/ensindexer/src/lib/indexing-engines/ponder.test.ts @@ -7,7 +7,26 @@ import type { IndexingEngineContext, IndexingEngineEvent } from "./ponder"; const { mockPonderOn } = vi.hoisted(() => ({ mockPonderOn: vi.fn() })); -const mockWaitForEnsRainbow = vi.hoisted(() => vi.fn()); +const { mockInitIndexingOnchainEvents } = vi.hoisted(() => ({ + mockInitIndexingOnchainEvents: vi.fn(), +})); + +// Set up PONDER_COMMON global before any imports that depend on it +vi.hoisted(() => { + (globalThis as any).PONDER_COMMON = { + options: { + command: "start", + port: 42069, + }, + logger: { + trace: vi.fn(), + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, + }; +}); vi.mock("ponder:registry", () => ({ ponder: { @@ -19,14 +38,14 @@ vi.mock("ponder:schema", () => ({ ensIndexerSchema: {}, })); -vi.mock("@/lib/ensrainbow/singleton", () => ({ - waitForEnsRainbowToBeReady: mockWaitForEnsRainbow, +vi.mock("./init-indexing-onchain-events", () => ({ + initIndexingOnchainEvents: mockInitIndexingOnchainEvents, })); describe("addOnchainEventListener", () => { beforeEach(async () => { vi.clearAllMocks(); - mockWaitForEnsRainbow.mockResolvedValue(undefined); + mockInitIndexingOnchainEvents.mockResolvedValue(undefined); // Reset module state to test idempotent behavior correctly vi.resetModules(); }); @@ -223,8 +242,8 @@ describe("addOnchainEventListener", () => { }); }); - describe("ENSRainbow preconditions (onchain events)", () => { - it("waits for ENSRainbow before executing the handler", async () => { + describe("onchain event preconditions", () => { + it("runs onchain event initialization before executing the handler", async () => { const { addOnchainEventListener } = await getPonderModule(); const handler = vi.fn().mockResolvedValue(undefined); @@ -234,14 +253,16 @@ describe("addOnchainEventListener", () => { event: {} as IndexingEngineEvent, }); - expect(mockWaitForEnsRainbow).toHaveBeenCalledTimes(1); + expect(mockInitIndexingOnchainEvents).toHaveBeenCalledTimes(1); expect(handler).toHaveBeenCalled(); }); - it("prevents handler execution if ENSRainbow is not ready", async () => { + it("prevents handler execution if onchain event initialization fails", async () => { const { addOnchainEventListener } = await getPonderModule(); const handler = vi.fn().mockResolvedValue(undefined); - mockWaitForEnsRainbow.mockRejectedValue(new Error("ENSRainbow not ready")); + mockInitIndexingOnchainEvents.mockRejectedValue( + new Error("Onchain event initialization failed"), + ); addOnchainEventListener("Resolver:AddrChanged" as EventNames, handler); @@ -250,12 +271,12 @@ describe("addOnchainEventListener", () => { context: { db: vi.fn() } as unknown as Context, event: {} as IndexingEngineEvent, }), - ).rejects.toThrow("ENSRainbow not ready"); + ).rejects.toThrow("Onchain event initialization failed"); expect(handler).not.toHaveBeenCalled(); }); - it("calls waitForEnsRainbowToBeReady only once across multiple onchain events (idempotent)", async () => { + it("calls initIndexingOnchainEvents only once across multiple onchain events (idempotent)", async () => { const { addOnchainEventListener } = await getPonderModule(); const handler1 = vi.fn().mockResolvedValue(undefined); const handler2 = vi.fn().mockResolvedValue(undefined); @@ -269,7 +290,7 @@ describe("addOnchainEventListener", () => { context: { db: vi.fn() } as unknown as Context, event: { args: { a: "1" } } as unknown as IndexingEngineEvent, }); - expect(mockWaitForEnsRainbow).toHaveBeenCalledTimes(1); + expect(mockInitIndexingOnchainEvents).toHaveBeenCalledTimes(1); // Trigger the second event handler await getRegisteredCallback(1)({ @@ -278,19 +299,19 @@ describe("addOnchainEventListener", () => { }); // Should still only have been called once (idempotent behavior) - expect(mockWaitForEnsRainbow).toHaveBeenCalledTimes(1); + expect(mockInitIndexingOnchainEvents).toHaveBeenCalledTimes(1); expect(handler1).toHaveBeenCalledTimes(1); expect(handler2).toHaveBeenCalledTimes(1); }); - it("calls waitForEnsRainbowToBeReady only once when two onchain callbacks fire concurrently before the readiness promise resolves", async () => { + it("calls initIndexingOnchainEvents only once when two onchain callbacks fire concurrently before the initialization promise resolves", async () => { const { addOnchainEventListener } = await getPonderModule(); const handler1 = vi.fn().mockResolvedValue(undefined); const handler2 = vi.fn().mockResolvedValue(undefined); let resolveReadiness: (() => void) | undefined; // Create a promise that won't resolve until we manually trigger it - mockWaitForEnsRainbow.mockImplementation(() => { + mockInitIndexingOnchainEvents.mockImplementation(() => { return new Promise((resolve) => { resolveReadiness = resolve; }); @@ -310,8 +331,8 @@ describe("addOnchainEventListener", () => { event: { args: { a: "2" } } as unknown as IndexingEngineEvent, }); - // Should only have been called once despite concurrent execution - expect(mockWaitForEnsRainbow).toHaveBeenCalledTimes(1); + // Allow the dynamic import to settle before asserting + await vi.waitFor(() => expect(mockInitIndexingOnchainEvents).toHaveBeenCalledTimes(1)); // Neither handler should have executed yet expect(handler1).not.toHaveBeenCalled(); @@ -328,12 +349,12 @@ describe("addOnchainEventListener", () => { expect(handler2).toHaveBeenCalledTimes(1); }); - it("resolves ENSRainbow before calling the handler", async () => { + it("resolves onchain event initialization before calling the handler", async () => { const { addOnchainEventListener } = await getPonderModule(); const handler = vi.fn().mockResolvedValue(undefined); let preconditionResolved = false; - mockWaitForEnsRainbow.mockImplementation(async () => { + mockInitIndexingOnchainEvents.mockImplementation(async () => { await new Promise((resolve) => setTimeout(resolve, 10)); preconditionResolved = true; }); @@ -350,7 +371,7 @@ describe("addOnchainEventListener", () => { }); describe("setup events (no preconditions)", () => { - it("skips ENSRainbow wait for :setup events", async () => { + it("skips onchain event initialization for :setup events", async () => { const { addOnchainEventListener } = await getPonderModule(); const handler = vi.fn().mockResolvedValue(undefined); @@ -360,7 +381,7 @@ describe("addOnchainEventListener", () => { event: {} as IndexingEngineEvent, }); - expect(mockWaitForEnsRainbow).not.toHaveBeenCalled(); + expect(mockInitIndexingOnchainEvents).not.toHaveBeenCalled(); expect(handler).toHaveBeenCalled(); }); @@ -383,7 +404,7 @@ describe("addOnchainEventListener", () => { event: {} as IndexingEngineEvent, }); - expect(mockWaitForEnsRainbow).not.toHaveBeenCalled(); + expect(mockInitIndexingOnchainEvents).not.toHaveBeenCalled(); expect(handler).toHaveBeenCalled(); } }); @@ -398,20 +419,20 @@ describe("addOnchainEventListener", () => { addOnchainEventListener("PublicResolver:setup" as EventNames, setupHandler); addOnchainEventListener("PublicResolver:AddrChanged" as EventNames, onchainHandler); - // Setup event - no ENSRainbow wait + // Setup event - no onchain event initialization await getRegisteredCallback(0)({ context: { db: vi.fn() } as unknown as Context, event: {} as IndexingEngineEvent, }); - expect(mockWaitForEnsRainbow).not.toHaveBeenCalled(); + expect(mockInitIndexingOnchainEvents).not.toHaveBeenCalled(); expect(setupHandler).toHaveBeenCalled(); - // Onchain event - ENSRainbow wait required + // Onchain event - initialization required await getRegisteredCallback(1)({ context: { db: vi.fn() } as unknown as Context, event: {} as IndexingEngineEvent, }); - expect(mockWaitForEnsRainbow).toHaveBeenCalledTimes(1); + expect(mockInitIndexingOnchainEvents).toHaveBeenCalledTimes(1); expect(onchainHandler).toHaveBeenCalled(); }); @@ -436,7 +457,7 @@ describe("addOnchainEventListener", () => { event: {} as IndexingEngineEvent, }); - expect(mockWaitForEnsRainbow).toHaveBeenCalled(); + expect(mockInitIndexingOnchainEvents).toHaveBeenCalled(); expect(handler).toHaveBeenCalled(); } }); diff --git a/apps/ensindexer/src/lib/indexing-engines/ponder.ts b/apps/ensindexer/src/lib/indexing-engines/ponder.ts index 5678242732..4dfda43f62 100644 --- a/apps/ensindexer/src/lib/indexing-engines/ponder.ts +++ b/apps/ensindexer/src/lib/indexing-engines/ponder.ts @@ -1,9 +1,48 @@ /** - * This module is an abstraction layer for the Indexing Engine of ENSIndexer. - * It decouples core indexing logic from Ponder-specific implementation details. - * Benefits of this decoupling include: - * - Building a custom context data model. - * - Implementing shared logic before or after event handlers, if needed. + * Ponder Indexing Engine + * + * This module provides an abstraction layer over the Ponder Indexing Engine + * to decouple the core indexing logic of the ENSIndexer from Ponder-specific + * implementation details. This allows us to build a custom context data model, + * and implement shared logic before or after event handlers, if needed, without + * affecting the "hot path" of indexing onchain events. + * + * Ponder Indexing Engine runs within an ENSIndexer instance, and is responsible + * for: + * - Managing the Ponder Schema and the ENSIndexer Schema in ENSDb instance. + * - Running HTTP server. + * - Executing the omnichain indexing strategy for sourcing and handling events. + * + * The startup sequence of the Ponder Indexing Engine is as follows: + * 1. Execute Ponder Config file (apps/ensindexer/ponder/ponder.config.ts), + * Ponder Schema file (apps/ensindexer/ponder/ponder.schema.ts), + * and all event handler files (as per nested imports in + * apps/ensindexer/ponder/src/register-handlers.ts). + * 2. Connect to the database and initialize required database objects. + * a) Execute database migrations for the ENSIndexer Schema in ENSDb. + * b) Execute database migrations for the Ponder Schema in ENSDb. + * 3. Execute Ponder HTTP API file (apps/ensindexer/ponder/src/api/index.ts) + * and start the HTTP server. + * 4. Execute the omnichain indexing strategy + * a) Start sourcing onchain events from the configured RPCs for + * the indexed contracts. + * b) Check if Ponder Checkpoints have been initialized in + * the ENSIndexer Schema in ENSDb. If not, execute + * the setup event handlers (if any), and initialize + * the Ponder Checkpoints in ENSDb. + * c) Once the Ponder Checkpoints are initialized, start executing + * the onchain event handlers for the sourced onchain events. + * + * The ENSIndexer instance has to be able to execute arbitrary logic + * before any onchain event handlers are executed, for example, to set up + * necessary state in the ENSNode Schema in ENSDb instance. To achieve this, + * we define the {@link addOnchainEventListener} function, which is + * a thin wrapper around {@link ponder.on} that allows us to execute additional + * logic before the onchain event handlers are executed, while keeping the + * "hot path" of indexing onchain events as efficient as possible. + * + * For more details on Ponder and its concepts, see the Ponder documentation. + * @see https://ponder.sh/docs/indexing/overview */ export * as ensIndexerSchema from "ponder:schema"; @@ -15,7 +54,6 @@ import { ponder, } from "ponder:registry"; -import { waitForEnsRainbowToBeReady } from "@/lib/ensrainbow/singleton"; import { logger } from "@/lib/logger"; /** @@ -114,7 +152,7 @@ const EventTypeIds = { * * Driven by an onchain event emitted by an indexed contract. */ - Onchain: "Onchain", + OnchainEvent: "OnchainEvent", } as const; /** @@ -126,59 +164,11 @@ function buildEventTypeId(eventName: EventNames): EventTypeId { if (eventName.endsWith(":setup")) { return EventTypeIds.Setup; } else { - return EventTypeIds.Onchain; + return EventTypeIds.OnchainEvent; } } -/** - * Prepare for executing the "setup" event handlers. - * - * During Ponder startup, the "setup" event handlers are executed: - * - After Ponder completed database migrations for ENSIndexer Schema in ENSDb. - * - Before Ponder starts processing any onchain events for indexed chains. - * - * This function is useful to make sure ENSDb is ready for writes, for example, - * by ensuring all required Postgres extensions are installed, etc. - */ -async function initializeIndexingSetup(): Promise { - /** - * Setup event handlers should not have any *long-running* preconditions. This is because - * Ponder populates the indexing metrics for all indexed chains only after all setup handlers have run. - * ENSIndexer relies on these indexing metrics being immediately available on startup to build and - * store the current Indexing Status in ENSDb. - */ -} - -/** - * Prepare for executing the "onchain" event handlers. - * - * During Ponder startup, the "onchain" event handlers are executed - * after all "setup" event handlers have completed. - * - * This function is useful to make sure any long-running preconditions for - * onchain event handlers are met, for example, waiting for - * the ENSRainbow instance to be ready before processing any onchain events - * that require data from ENSRainbow. - * - * @example A single blocking precondition - * ```ts - * await waitForEnsRainbowToBeReady(); - * ``` - * - * @example Multiple blocking preconditions - * ```ts - * await Promise.all([ - * waitForEnsRainbowToBeReady(), - * waitForAnotherPrecondition(), - * ]); - * ``` - */ -async function initializeIndexingActivation(): Promise { - await waitForEnsRainbowToBeReady(); -} - -let indexingSetupPromise: Promise | null = null; -let indexingActivationPromise: Promise | null = null; +let initIndexingOnchainEventsPromise: Promise | null = null; // Cumulative events-per-second tracking across the process lifetime. Logged at most // once per minute. Overhead is one Date.now() and a counter increment per event. @@ -212,7 +202,7 @@ function recordEventForEps(): void { * Some event handlers may have preconditions that need to be met before * they can run. * - * The Setup and Onchain preconditions are memoized and execute their logic only + * The Onchain preconditions are memoized and execute their logic only * once per process, regardless of how often this function is called — essential * because it's invoked for every indexed event. EPS accounting via * {@link recordEventForEps} runs on every call, but its hot-path cost is a @@ -224,23 +214,31 @@ async function eventHandlerPreconditions(eventType: EventTypeId): Promise switch (eventType) { case EventTypeIds.Setup: { - if (indexingSetupPromise === null) { - // Initialize the indexing setup just once. - indexingSetupPromise = initializeIndexingSetup(); - } - - return await indexingSetupPromise; + // For some ENSIndexer instances, the setup handlers are not defined at all, + // for example, if the ENSIndexer instance has only the `ensv2` plugin activated. + // In this case, some important logic, such as running migrations for ENSNode Schema + // in ENSDb, would not be executed at all, which would cause the ENSIndexer instance + // to not work properly. Therefore, all logic required to be executed before + // indexing of onchain events should be executed in initIndexingOnchainEvents function. + return; } - case EventTypeIds.Onchain: { - if (indexingActivationPromise === null) { - // Initialize the indexing activation just once in order to - // optimize the "hot path" of indexing onchain events, since these are - // much more frequent than setup events. - indexingActivationPromise = initializeIndexingActivation(); + case EventTypeIds.OnchainEvent: { + if (initIndexingOnchainEventsPromise === null) { + // We need to work around the Ponder limitation for importing modules, + // since Ponder would not allow us to use static imports for modules + // that internally rely on `ponder:api`. Using dynamic imports solves + // this issue. + initIndexingOnchainEventsPromise = import("./init-indexing-onchain-events").then( + ({ initIndexingOnchainEvents }) => + // Init the indexing of "onchain" events just once in order to + // optimize the indexing "hot path", since these events are much + // more frequent than setup events. + initIndexingOnchainEvents(), + ); } - return await indexingActivationPromise; + return await initIndexingOnchainEventsPromise; } } } diff --git a/apps/ensindexer/src/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.mock.ts b/apps/ensindexer/src/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.mock.ts new file mode 100644 index 0000000000..0f85c30930 --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.mock.ts @@ -0,0 +1,101 @@ +import { vi } from "vitest"; + +import type { EnsDbReader } from "@ensnode/ensdb-sdk"; +import { + type CrossChainIndexingStatusSnapshot, + type EnsIndexerStackInfo, + type IndexingMetadataContext, + type IndexingMetadataContextInitialized, + IndexingMetadataContextStatusCodes, + OmnichainIndexingStatusIds, + type OmnichainIndexingStatusSnapshot, +} from "@ensnode/ensnode-sdk"; +import type { LocalPonderClient } from "@ensnode/ponder-sdk"; + +import { IndexingMetadataContextBuilder } from "@/lib/indexing-metadata-context-builder/indexing-metadata-context-builder"; +import type { IndexingStatusBuilder } from "@/lib/indexing-status-builder"; +import type { StackInfoBuilder } from "@/lib/stack-info-builder"; + +export const omnichainSnapshotUnstarted: OmnichainIndexingStatusSnapshot = { + omnichainStatus: OmnichainIndexingStatusIds.Unstarted, + omnichainIndexingCursor: 0, + chains: new Map(), +}; + +export const omnichainSnapshotFollowing: OmnichainIndexingStatusSnapshot = { + omnichainStatus: OmnichainIndexingStatusIds.Following, + omnichainIndexingCursor: 100, + chains: new Map(), +}; + +export const crossChainSnapshot: CrossChainIndexingStatusSnapshot = { + strategy: "omnichain" as any, + slowestChainIndexingCursor: 100, + snapshotTime: 200, + omnichainSnapshot: omnichainSnapshotFollowing, +}; + +export const stackInfo: EnsIndexerStackInfo = { + ensDb: { versionInfo: { postgresql: "17.4" } }, + ensIndexer: {} as any, + ensRainbow: {} as any, +}; + +export const indexingMetadataContextInitialized: IndexingMetadataContextInitialized = { + statusCode: IndexingMetadataContextStatusCodes.Initialized, + indexingStatus: crossChainSnapshot, + stackInfo, +}; + +export const indexingMetadataContextUninitialized: IndexingMetadataContext = { + statusCode: IndexingMetadataContextStatusCodes.Uninitialized, +}; + +export function createMockEnsDbReader( + overrides: Partial> = {}, +): EnsDbReader { + return { + getIndexingMetadataContext: vi.fn().mockResolvedValue(indexingMetadataContextUninitialized), + ...overrides, + } as unknown as EnsDbReader; +} + +export function createMockIndexingStatusBuilder( + resolvedSnapshot: OmnichainIndexingStatusSnapshot = omnichainSnapshotUnstarted, +): IndexingStatusBuilder { + return { + getOmnichainIndexingStatusSnapshot: vi.fn().mockResolvedValue(resolvedSnapshot), + } as unknown as IndexingStatusBuilder; +} + +export function createMockStackInfoBuilder( + resolvedStackInfo: EnsIndexerStackInfo = stackInfo, +): StackInfoBuilder { + return { + getStackInfo: vi.fn().mockResolvedValue(resolvedStackInfo), + } as unknown as StackInfoBuilder; +} + +export function createMockLocalPonderClient( + options: { isInDevMode?: boolean } = {}, +): LocalPonderClient { + return { + isInDevMode: options.isInDevMode ?? false, + } as unknown as LocalPonderClient; +} + +export function createIndexingMetadataContextBuilder( + overrides: { + ensDbClient?: EnsDbReader; + indexingStatusBuilder?: IndexingStatusBuilder; + stackInfoBuilder?: StackInfoBuilder; + localPonderClient?: LocalPonderClient; + } = {}, +): IndexingMetadataContextBuilder { + return new IndexingMetadataContextBuilder( + overrides.ensDbClient ?? createMockEnsDbReader(), + overrides.indexingStatusBuilder ?? createMockIndexingStatusBuilder(), + overrides.stackInfoBuilder ?? createMockStackInfoBuilder(), + overrides.localPonderClient ?? createMockLocalPonderClient(), + ); +} diff --git a/apps/ensindexer/src/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.test.ts b/apps/ensindexer/src/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.test.ts new file mode 100644 index 0000000000..24020155bb --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.test.ts @@ -0,0 +1,217 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { + buildCrossChainIndexingStatusSnapshotOmnichain, + buildIndexingMetadataContextInitialized, + type IndexingMetadataContextInitialized, + validateEnsIndexerPublicConfigCompatibility, +} from "@ensnode/ensnode-sdk"; + +import "@/lib/__test__/mockLogger"; + +import { + createIndexingMetadataContextBuilder, + createMockEnsDbReader, + createMockIndexingStatusBuilder, + createMockLocalPonderClient, + createMockStackInfoBuilder, + crossChainSnapshot, + indexingMetadataContextInitialized, + indexingMetadataContextUninitialized, + omnichainSnapshotFollowing, + omnichainSnapshotUnstarted, + stackInfo, +} from "@/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.mock"; + +vi.mock("@ensnode/ensnode-sdk", async () => { + const actual = await vi.importActual("@ensnode/ensnode-sdk"); + + return { + ...actual, + buildCrossChainIndexingStatusSnapshotOmnichain: vi.fn(), + buildIndexingMetadataContextInitialized: vi.fn(), + validateEnsIndexerPublicConfigCompatibility: vi.fn(), + }; +}); + +describe("IndexingMetadataContextBuilder", () => { + beforeEach(() => { + vi.clearAllMocks(); + + vi.mocked(buildCrossChainIndexingStatusSnapshotOmnichain).mockReturnValue(crossChainSnapshot); + vi.mocked(buildIndexingMetadataContextInitialized).mockReturnValue( + indexingMetadataContextInitialized as IndexingMetadataContextInitialized, + ); + }); + + describe("getIndexingMetadataContext()", () => { + describe("when stored context is Uninitialized", () => { + it("builds and returns initialized context with fresh snapshot time", async () => { + const ensDbClient = createMockEnsDbReader(); + const indexingStatusBuilder = createMockIndexingStatusBuilder(omnichainSnapshotUnstarted); + const stackInfoBuilder = createMockStackInfoBuilder(); + + const builder = createIndexingMetadataContextBuilder({ + ensDbClient, + indexingStatusBuilder, + stackInfoBuilder, + }); + const result = await builder.getIndexingMetadataContext(); + + expect(ensDbClient.getIndexingMetadataContext).toHaveBeenCalledOnce(); + expect(indexingStatusBuilder.getOmnichainIndexingStatusSnapshot).toHaveBeenCalledOnce(); + expect(stackInfoBuilder.getStackInfo).toHaveBeenCalledOnce(); + expect(buildCrossChainIndexingStatusSnapshotOmnichain).toHaveBeenCalledWith( + omnichainSnapshotUnstarted, + expect.any(Number), + ); + expect(buildIndexingMetadataContextInitialized).toHaveBeenCalledWith( + crossChainSnapshot, + stackInfo, + ); + expect(result).toBe(indexingMetadataContextInitialized); + }); + + it("throws when indexing status is not unstarted", async () => { + const indexingStatusBuilder = createMockIndexingStatusBuilder(omnichainSnapshotFollowing); + + const builder = createIndexingMetadataContextBuilder({ + indexingStatusBuilder, + }); + + await expect(builder.getIndexingMetadataContext()).rejects.toThrow( + /Omnichain indexing status must be "unstarted"/, + ); + }); + }); + + describe("when stored context is Initialized", () => { + it("validates compatibility when not in dev mode", async () => { + const ensDbClient = createMockEnsDbReader({ + getIndexingMetadataContext: vi.fn().mockResolvedValue(indexingMetadataContextInitialized), + }); + const indexingStatusBuilder = createMockIndexingStatusBuilder(omnichainSnapshotFollowing); + const stackInfoBuilder = createMockStackInfoBuilder(); + const localPonderClient = createMockLocalPonderClient({ isInDevMode: false }); + + const builder = createIndexingMetadataContextBuilder({ + ensDbClient, + indexingStatusBuilder, + stackInfoBuilder, + localPonderClient, + }); + const result = await builder.getIndexingMetadataContext(); + + expect(validateEnsIndexerPublicConfigCompatibility).toHaveBeenCalledWith( + (indexingMetadataContextInitialized as IndexingMetadataContextInitialized).stackInfo + .ensIndexer, + stackInfo.ensIndexer, + ); + expect(buildIndexingMetadataContextInitialized).toHaveBeenCalledWith( + crossChainSnapshot, + stackInfo, + ); + expect(result).toBe(indexingMetadataContextInitialized); + }); + + it("skips compatibility validation when in dev mode", async () => { + const ensDbClient = createMockEnsDbReader({ + getIndexingMetadataContext: vi.fn().mockResolvedValue(indexingMetadataContextInitialized), + }); + const indexingStatusBuilder = createMockIndexingStatusBuilder(omnichainSnapshotFollowing); + const stackInfoBuilder = createMockStackInfoBuilder(); + const localPonderClient = createMockLocalPonderClient({ isInDevMode: true }); + + const builder = createIndexingMetadataContextBuilder({ + ensDbClient, + indexingStatusBuilder, + stackInfoBuilder, + localPonderClient, + }); + const result = await builder.getIndexingMetadataContext(); + + // Compatibility validation should NOT be called in dev mode + expect(validateEnsIndexerPublicConfigCompatibility).not.toHaveBeenCalled(); + expect(buildIndexingMetadataContextInitialized).toHaveBeenCalledWith( + crossChainSnapshot, + stackInfo, + ); + expect(result).toBe(indexingMetadataContextInitialized); + }); + + it("throws when stored and in-memory configs are incompatible (not in dev mode)", async () => { + vi.mocked(validateEnsIndexerPublicConfigCompatibility).mockImplementation(() => { + throw new Error("Incompatible ENSIndexer config"); + }); + + const ensDbClient = createMockEnsDbReader({ + getIndexingMetadataContext: vi.fn().mockResolvedValue(indexingMetadataContextInitialized), + }); + + const builder = createIndexingMetadataContextBuilder({ + ensDbClient, + indexingStatusBuilder: createMockIndexingStatusBuilder(omnichainSnapshotFollowing), + localPonderClient: createMockLocalPonderClient({ isInDevMode: false }), + }); + + await expect(builder.getIndexingMetadataContext()).rejects.toThrow( + "Incompatible ENSIndexer config", + ); + }); + + it("does not throw on incompatible configs when in dev mode", async () => { + vi.mocked(validateEnsIndexerPublicConfigCompatibility).mockImplementation(() => { + throw new Error("Incompatible ENSIndexer config"); + }); + + const ensDbClient = createMockEnsDbReader({ + getIndexingMetadataContext: vi.fn().mockResolvedValue(indexingMetadataContextInitialized), + }); + + const builder = createIndexingMetadataContextBuilder({ + ensDbClient, + indexingStatusBuilder: createMockIndexingStatusBuilder(omnichainSnapshotFollowing), + localPonderClient: createMockLocalPonderClient({ isInDevMode: true }), + }); + + await expect(builder.getIndexingMetadataContext()).resolves.toBeDefined(); + }); + }); + + it("fetches all three data sources in parallel", async () => { + const resolveOrder: string[] = []; + const ensDbClient = createMockEnsDbReader({ + getIndexingMetadataContext: vi.fn().mockImplementation(async () => { + await new Promise((r) => setTimeout(r, 10)); + resolveOrder.push("ensDb"); + return indexingMetadataContextUninitialized; + }), + }); + const indexingStatusBuilder = createMockIndexingStatusBuilder(omnichainSnapshotUnstarted); + (indexingStatusBuilder.getOmnichainIndexingStatusSnapshot as any) = vi + .fn() + .mockImplementation(async () => { + resolveOrder.push("indexingStatus"); + return omnichainSnapshotUnstarted; + }); + const stackInfoBuilder = createMockStackInfoBuilder(); + (stackInfoBuilder.getStackInfo as any) = vi.fn().mockImplementation(async () => { + resolveOrder.push("stackInfo"); + return stackInfo; + }); + + const builder = createIndexingMetadataContextBuilder({ + ensDbClient, + indexingStatusBuilder, + stackInfoBuilder, + }); + await builder.getIndexingMetadataContext(); + + // All three should have been called (ordering is not deterministic for parallel) + expect(resolveOrder).toHaveLength(3); + expect(resolveOrder).toContain("ensDb"); + expect(resolveOrder).toContain("indexingStatus"); + expect(resolveOrder).toContain("stackInfo"); + }); + }); +}); diff --git a/apps/ensindexer/src/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.ts b/apps/ensindexer/src/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.ts new file mode 100644 index 0000000000..1850a76c76 --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-metadata-context-builder/indexing-metadata-context-builder.ts @@ -0,0 +1,142 @@ +import { getUnixTime } from "date-fns"; + +import type { EnsDbReader } from "@ensnode/ensdb-sdk"; +import { + buildCrossChainIndexingStatusSnapshotOmnichain, + buildIndexingMetadataContextInitialized, + type EnsIndexerStackInfo, + type IndexingMetadataContextInitialized, + IndexingMetadataContextStatusCodes, + OmnichainIndexingStatusIds, + type OmnichainIndexingStatusSnapshot, + validateEnsIndexerPublicConfigCompatibility, +} from "@ensnode/ensnode-sdk"; +import type { LocalPonderClient } from "@ensnode/ponder-sdk"; + +import type { IndexingStatusBuilder } from "@/lib/indexing-status-builder/indexing-status-builder"; +import { logger } from "@/lib/logger"; +import type { StackInfoBuilder } from "@/lib/stack-info-builder/stack-info-builder"; + +function invariant_indexingStatusIsUnstartedForIndexingMetadataContextUninitialized( + inMemoryIndexingStatusSnapshot: OmnichainIndexingStatusSnapshot, +): void { + if (inMemoryIndexingStatusSnapshot.omnichainStatus !== OmnichainIndexingStatusIds.Unstarted) { + throw new Error( + `Omnichain indexing status must be "unstarted" for "uninitialized" Indexing Metadata Context. Provided omnichain indexing status "${inMemoryIndexingStatusSnapshot.omnichainStatus}".`, + ); + } +} + +function invariant_ensIndexerPublicConfigIsCompatibleWithStackInfo( + storedEnsIndexerStackInfo: EnsIndexerStackInfo, + inMemoryEnsIndexerStackInfo: EnsIndexerStackInfo, +): void { + const { ensIndexer: storedEnsIndexerPublicConfig } = storedEnsIndexerStackInfo; + const { ensIndexer: inMemoryEnsIndexerPublicConfig } = inMemoryEnsIndexerStackInfo; + + validateEnsIndexerPublicConfigCompatibility( + storedEnsIndexerPublicConfig, + inMemoryEnsIndexerPublicConfig, + ); +} + +export class IndexingMetadataContextBuilder { + constructor( + /** + * ENSDb Client used to read the currently stored + * {@link IndexingMetadataContextInitialized} record from ENSDb, + * which the invariant validation logic in + * {@link getIndexingMetadataContext} depends on. + */ + private readonly ensDbClient: EnsDbReader, + + /** + * IndexingStatusBuilder used to get + * the current in-memory {@link OmnichainIndexingStatusSnapshot} for building + * the "in-memory" {@link IndexingMetadataContextInitialized} object + * within {@link getIndexingMetadataContext}. + */ + private readonly indexingStatusBuilder: IndexingStatusBuilder, + + /** + * StackInfoBuilder used to get + * the current in-memory {@link EnsIndexerStackInfo} for building + * the "in-memory" {@link IndexingMetadataContextInitialized} object + * within {@link getIndexingMetadataContext}. + */ + private readonly stackInfoBuilder: StackInfoBuilder, + + /** + * Local Ponder Client used to determine if the local Ponder app + * is running in dev mode, which affects the validation logic applied + * in {@link getIndexingMetadataContext}. + */ + private readonly localPonderClient: LocalPonderClient, + ) {} + + /** + * Get the current {@link IndexingMetadataContextInitialized} object. + * + * Expected to be called while writing the {@link IndexingMetadataContextInitialized} record in ENSDb. + */ + async getIndexingMetadataContext(): Promise { + const [ + inMemoryIndexingStatusSnapshot, + inMemoryEnsIndexerStackInfo, + storedIndexingMetadataContext, + ] = await Promise.all([ + this.indexingStatusBuilder.getOmnichainIndexingStatusSnapshot(), + this.stackInfoBuilder.getStackInfo(), + this.ensDbClient.getIndexingMetadataContext(), + ]); + + // Build the CrossChainIndexingStatusSnapshot with the current snapshot time. + // This is important to make sure the `snapshotTime` is always up to date in + // the indexing status snapshot stored within the Indexing Metadata Context record in ENSDb. + const now = getUnixTime(new Date()); + const crossChainIndexingStatusSnapshot = buildCrossChainIndexingStatusSnapshotOmnichain( + inMemoryIndexingStatusSnapshot, + now, + ); + + const inMemoryIndexingMetadataContext = buildIndexingMetadataContextInitialized( + crossChainIndexingStatusSnapshot, + inMemoryEnsIndexerStackInfo, + ); + + if ( + storedIndexingMetadataContext.statusCode === IndexingMetadataContextStatusCodes.Uninitialized + ) { + logger.info({ msg: `Indexing Metadata Context is "uninitialized"` }); + + // If no IndexingMetadataContext has been initialized in ENSDb yet, then + // the "in-memory" CrossChainIndexingStatusSnapshot must be in + // "unstarted" status, since onchain events indexing has not started yet. + invariant_indexingStatusIsUnstartedForIndexingMetadataContextUninitialized( + inMemoryIndexingStatusSnapshot, + ); + } else { + logger.debug({ msg: `Indexing Metadata Context is "initialized"` }); + logger.trace({ + msg: `Indexing Metadata Context`, + indexingStatus: storedIndexingMetadataContext.indexingStatus, + stackInfo: storedIndexingMetadataContext.stackInfo, + }); + + // For EnsIndexerPublicConfig, validate in-memory config object + // compatibility with the stored one, if the stored one is available. + // The validation is skipped if the local Ponder app is running in dev mode. + // This is to improve the development experience during ENSIndexer + // development, by allowing to override the stored config in ENSDb with + // the current in-memory config, without having to keep them compatible. + if (!this.localPonderClient.isInDevMode) { + invariant_ensIndexerPublicConfigIsCompatibleWithStackInfo( + storedIndexingMetadataContext.stackInfo, + inMemoryEnsIndexerStackInfo, + ); + } + } + + return inMemoryIndexingMetadataContext; + } +} diff --git a/apps/ensindexer/src/lib/indexing-metadata-context-builder/singleton.ts b/apps/ensindexer/src/lib/indexing-metadata-context-builder/singleton.ts new file mode 100644 index 0000000000..31299cd55f --- /dev/null +++ b/apps/ensindexer/src/lib/indexing-metadata-context-builder/singleton.ts @@ -0,0 +1,15 @@ +import { ensDbClient } from "@/lib/ensdb/singleton"; +import { IndexingMetadataContextBuilder } from "@/lib/indexing-metadata-context-builder/indexing-metadata-context-builder"; +import { indexingStatusBuilder } from "@/lib/indexing-status-builder/singleton"; +import { localPonderClient } from "@/lib/local-ponder-client"; +import { stackInfoBuilder } from "@/lib/stack-info-builder/singleton"; + +/** + * Singleton {@link IndexingMetadataContextBuilder} instance to use across ENSIndexer modules. + */ +export const indexingMetadataContextBuilder = new IndexingMetadataContextBuilder( + ensDbClient, + indexingStatusBuilder, + stackInfoBuilder, + localPonderClient, +); diff --git a/apps/ensindexer/src/lib/public-config-builder/public-config-builder.ts b/apps/ensindexer/src/lib/public-config-builder/public-config-builder.ts index 7229af0367..9b132ddcee 100644 --- a/apps/ensindexer/src/lib/public-config-builder/public-config-builder.ts +++ b/apps/ensindexer/src/lib/public-config-builder/public-config-builder.ts @@ -37,16 +37,19 @@ export class PublicConfigBuilder { /** * Get ENSIndexer Public Config * - * Note: ENSIndexer Public Config is cached after the first call, so - * subsequent calls will return the cached version without rebuilding it. + * Note: The {@link EnsIndexerPublicConfig} object is immutable for + * the whole ENSIndexer instance lifecycle. Therefore, the result of + * the first {@link getPublicConfig} call is cached and returned for + * subsequent calls. * - * @throws if the built ENSIndexer Public Config does not conform to + * @throws if the built {@link EnsIndexerPublicConfig} does not conform to * the expected schema */ async getPublicConfig(): Promise { if (typeof this.immutablePublicConfig === "undefined") { const [versionInfo, ensRainbowPublicConfig] = await Promise.all([ this.getEnsIndexerVersionInfo(), + // TODO: remove dependency on ENSRainbow by dropping `ensRainbowPublicConfig` from `EnsIndexerPublicConfig`. this.ensRainbowClient.config(), ]); diff --git a/apps/ensindexer/src/lib/stack-info-builder/index.ts b/apps/ensindexer/src/lib/stack-info-builder/index.ts new file mode 100644 index 0000000000..9df1b93e90 --- /dev/null +++ b/apps/ensindexer/src/lib/stack-info-builder/index.ts @@ -0,0 +1 @@ +export * from "./stack-info-builder"; diff --git a/apps/ensindexer/src/lib/stack-info-builder/singleton.ts b/apps/ensindexer/src/lib/stack-info-builder/singleton.ts new file mode 100644 index 0000000000..2b924c0589 --- /dev/null +++ b/apps/ensindexer/src/lib/stack-info-builder/singleton.ts @@ -0,0 +1,13 @@ +import { ensDbClient } from "@/lib/ensdb/singleton"; +import { ensRainbowClient } from "@/lib/ensrainbow/singleton"; +import { publicConfigBuilder } from "@/lib/public-config-builder/singleton"; +import { StackInfoBuilder } from "@/lib/stack-info-builder/stack-info-builder"; + +/** + * Singleton {@link StackInfoBuilder} instance to use across ENSIndexer modules. + */ +export const stackInfoBuilder = new StackInfoBuilder( + ensDbClient, + ensRainbowClient, + publicConfigBuilder, +); diff --git a/apps/ensindexer/src/lib/stack-info-builder/stack-info-builder.test.ts b/apps/ensindexer/src/lib/stack-info-builder/stack-info-builder.test.ts new file mode 100644 index 0000000000..259233d199 --- /dev/null +++ b/apps/ensindexer/src/lib/stack-info-builder/stack-info-builder.test.ts @@ -0,0 +1,188 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { EnsDbReader } from "@ensnode/ensdb-sdk"; +import { + buildEnsIndexerStackInfo, + type EnsIndexerPublicConfig, + type EnsIndexerStackInfo, +} from "@ensnode/ensnode-sdk"; +import type { EnsRainbowApiClient } from "@ensnode/ensrainbow-sdk/client"; + +import type { PublicConfigBuilder } from "@/lib/public-config-builder"; + +import { StackInfoBuilder } from "./stack-info-builder"; + +vi.mock("@ensnode/ensnode-sdk", async () => { + const actual = await vi.importActual("@ensnode/ensnode-sdk"); + + return { + ...actual, + buildEnsIndexerStackInfo: vi.fn(), + }; +}); + +const mockEnsDbPublicConfig = { + versionInfo: { postgresql: "17.4" }, +}; + +const mockEnsIndexerPublicConfig = { + ensIndexerSchemaName: "ensindexer_0", + ensRainbowPublicConfig: { + serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, + versionInfo: { ensRainbow: "1.9.0" }, + }, + clientLabelSet: { labelSetId: "subgraph", labelSetVersion: 0 }, + indexedChainIds: new Set([1]), + isSubgraphCompatible: true, + namespace: "mainnet", + plugins: [], + versionInfo: { + ponder: "0.11.0", + ensDb: "1.0.0", + ensIndexer: "1.0.0", + ensNormalize: "1.0.0", + }, +} satisfies EnsIndexerPublicConfig; + +const mockEnsRainbowPublicConfig = { + serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, + versionInfo: { ensRainbow: "1.9.0" }, +}; + +const mockStackInfo = { + ensDb: mockEnsDbPublicConfig, + ensIndexer: mockEnsIndexerPublicConfig, + ensRainbow: mockEnsRainbowPublicConfig, +} satisfies EnsIndexerStackInfo; + +function createMockEnsDbReader( + overrides: Partial> = {}, +): EnsDbReader { + return { + buildEnsDbPublicConfig: vi.fn().mockResolvedValue(mockEnsDbPublicConfig), + ...overrides, + } as unknown as EnsDbReader; +} + +function createMockEnsRainbowClient( + overrides: Partial> = {}, +): EnsRainbowApiClient { + return { + config: vi.fn().mockResolvedValue(mockEnsRainbowPublicConfig), + ...overrides, + } as unknown as EnsRainbowApiClient; +} + +function createMockPublicConfigBuilder( + overrides: Partial> = {}, +): PublicConfigBuilder { + return { + getPublicConfig: vi.fn().mockResolvedValue(mockEnsIndexerPublicConfig), + ...overrides, + } as unknown as PublicConfigBuilder; +} + +describe("StackInfoBuilder", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe("getStackInfo()", () => { + it("builds stack info from ensDb, ensIndexer, and ensRainbow public configs", async () => { + vi.mocked(buildEnsIndexerStackInfo).mockReturnValue(mockStackInfo); + + const ensDbClient = createMockEnsDbReader(); + const ensRainbowClient = createMockEnsRainbowClient(); + const publicConfigBuilder = createMockPublicConfigBuilder(); + + const builder = new StackInfoBuilder(ensDbClient, ensRainbowClient, publicConfigBuilder); + const result = await builder.getStackInfo(); + + expect(ensDbClient.buildEnsDbPublicConfig).toHaveBeenCalledOnce(); + expect(publicConfigBuilder.getPublicConfig).toHaveBeenCalledOnce(); + expect(ensRainbowClient.config).toHaveBeenCalledOnce(); + expect(buildEnsIndexerStackInfo).toHaveBeenCalledWith( + mockEnsDbPublicConfig, + mockEnsIndexerPublicConfig, + mockEnsRainbowPublicConfig, + ); + expect(result).toBe(mockStackInfo); + }); + + it("caches stack info and returns the same value on subsequent calls", async () => { + vi.mocked(buildEnsIndexerStackInfo).mockReturnValue(mockStackInfo); + + const ensDbClient = createMockEnsDbReader(); + const ensRainbowClient = createMockEnsRainbowClient(); + const publicConfigBuilder = createMockPublicConfigBuilder(); + + const builder = new StackInfoBuilder(ensDbClient, ensRainbowClient, publicConfigBuilder); + + const result1 = await builder.getStackInfo(); + const result2 = await builder.getStackInfo(); + + expect(result1).toBe(result2); + // Underlying dependencies should only be called once due to caching + expect(ensDbClient.buildEnsDbPublicConfig).toHaveBeenCalledOnce(); + expect(publicConfigBuilder.getPublicConfig).toHaveBeenCalledOnce(); + expect(ensRainbowClient.config).toHaveBeenCalledOnce(); + expect(buildEnsIndexerStackInfo).toHaveBeenCalledOnce(); + }); + + it("throws when buildEnsIndexerStackInfo throws", async () => { + vi.mocked(buildEnsIndexerStackInfo).mockImplementation(() => { + throw new Error("Stack info validation failed"); + }); + + const builder = new StackInfoBuilder( + createMockEnsDbReader(), + createMockEnsRainbowClient(), + createMockPublicConfigBuilder(), + ); + + await expect(builder.getStackInfo()).rejects.toThrow("Stack info validation failed"); + }); + + it("propagates errors from ensDb client", async () => { + const ensDbClient = createMockEnsDbReader({ + buildEnsDbPublicConfig: vi.fn().mockRejectedValue(new Error("ENSDB connection failed")), + }); + + const builder = new StackInfoBuilder( + ensDbClient, + createMockEnsRainbowClient(), + createMockPublicConfigBuilder(), + ); + + await expect(builder.getStackInfo()).rejects.toThrow("ENSDB connection failed"); + }); + + it("propagates errors from ensRainbow client", async () => { + const ensRainbowClient = createMockEnsRainbowClient({ + config: vi.fn().mockRejectedValue(new Error("ENSRainbow not available")), + }); + + const builder = new StackInfoBuilder( + createMockEnsDbReader(), + ensRainbowClient, + createMockPublicConfigBuilder(), + ); + + await expect(builder.getStackInfo()).rejects.toThrow("ENSRainbow not available"); + }); + + it("propagates errors from public config builder", async () => { + const publicConfigBuilder = createMockPublicConfigBuilder({ + getPublicConfig: vi.fn().mockRejectedValue(new Error("Config retrieval failed")), + }); + + const builder = new StackInfoBuilder( + createMockEnsDbReader(), + createMockEnsRainbowClient(), + publicConfigBuilder, + ); + + await expect(builder.getStackInfo()).rejects.toThrow("Config retrieval failed"); + }); + }); +}); diff --git a/apps/ensindexer/src/lib/stack-info-builder/stack-info-builder.ts b/apps/ensindexer/src/lib/stack-info-builder/stack-info-builder.ts new file mode 100644 index 0000000000..4418829a74 --- /dev/null +++ b/apps/ensindexer/src/lib/stack-info-builder/stack-info-builder.ts @@ -0,0 +1,50 @@ +import type { EnsDbReader } from "@ensnode/ensdb-sdk"; +import { buildEnsIndexerStackInfo, type EnsIndexerStackInfo } from "@ensnode/ensnode-sdk"; +import type { EnsRainbowApiClient } from "@ensnode/ensrainbow-sdk/client"; + +import type { PublicConfigBuilder } from "@/lib/public-config-builder"; + +export class StackInfoBuilder { + /** + * Immutable {@link EnsIndexerStackInfo} + * + * The cached {@link EnsIndexerStackInfo} object, which is built and validated + * on the first call to `getStackInfo()`, and returned as-is on subsequent calls. + */ + private immutableStackInfo: EnsIndexerStackInfo | undefined; + + constructor( + private readonly ensDbClient: EnsDbReader, + private readonly ensRainbowClient: EnsRainbowApiClient, + private readonly publicConfigBuilder: PublicConfigBuilder, + ) {} + + /** + * Get ENSIndexer Stack Info + * + * Note: ENSIndexer Stack Info is cached after the first call, so + * subsequent calls will return the cached version without rebuilding it. + * + * @throws if the built ENSIndexer Stack Info does not conform to + * the expected schema + */ + async getStackInfo(): Promise { + if (typeof this.immutableStackInfo === "undefined") { + const [ensDbPublicConfig, ensIndexerPublicConfig, ensRainbowPublicConfig] = await Promise.all( + [ + this.ensDbClient.buildEnsDbPublicConfig(), + this.publicConfigBuilder.getPublicConfig(), + this.ensRainbowClient.config(), + ], + ); + + this.immutableStackInfo = buildEnsIndexerStackInfo( + ensDbPublicConfig, + ensIndexerPublicConfig, + ensRainbowPublicConfig, + ); + } + + return this.immutableStackInfo; + } +} diff --git a/packages/ensdb-sdk/src/client/ensdb-reader.test.ts b/packages/ensdb-sdk/src/client/ensdb-reader.test.ts index 3a488c4a86..53646768e4 100644 --- a/packages/ensdb-sdk/src/client/ensdb-reader.test.ts +++ b/packages/ensdb-sdk/src/client/ensdb-reader.test.ts @@ -1,17 +1,22 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { + buildEnsIndexerStackInfo, + buildIndexingMetadataContextInitialized, + buildIndexingMetadataContextUninitialized, deserializeCrossChainIndexingStatusSnapshot, - serializeEnsIndexerPublicConfig, + type EnsDbPublicConfig, + serializeIndexingMetadataContext, } from "@ensnode/ensnode-sdk"; import * as ensDbClientMock from "./ensdb-client.mock"; import { EnsDbReader } from "./ensdb-reader"; +const executeMock = vi.fn(); const whereMock = vi.fn(async () => [] as Array<{ value: unknown }>); const fromMock = vi.fn(() => ({ where: whereMock })); const selectMock = vi.fn(() => ({ from: fromMock })); -const drizzleClientMock = { select: selectMock } as any; +const drizzleClientMock = { select: selectMock, execute: executeMock } as any; vi.mock("drizzle-orm/node-postgres", () => ({ drizzle: vi.fn(() => drizzleClientMock), @@ -29,59 +34,188 @@ describe("EnsDbReader", () => { whereMock.mockClear(); fromMock.mockClear(); selectMock.mockClear(); + executeMock.mockClear(); }); - describe("getEnsDbVersion", () => { - it("returns undefined when no record exists", async () => { - const ensDbClient = createEnsDbReader(); - const { ensNodeSchema } = ensDbClient; + describe("getters", () => { + it("returns the ensDb drizzle client", () => { + const ensDbReader = createEnsDbReader(); + expect(ensDbReader.ensDb).toBe(drizzleClientMock); + }); - await expect(ensDbClient.getEnsDbVersion()).resolves.toBeUndefined(); + it("returns the ensIndexerSchema", () => { + const ensDbReader = createEnsDbReader(); + expect(ensDbReader.ensIndexerSchema).toBeDefined(); + }); - expect(selectMock).toHaveBeenCalledTimes(1); - expect(fromMock).toHaveBeenCalledWith(ensNodeSchema.metadata); + it("returns the ensIndexerSchemaName", () => { + const ensDbReader = createEnsDbReader(); + expect(ensDbReader.ensIndexerSchemaName).toBe(ensDbClientMock.ensIndexerSchemaName); }); - it("returns value when one record exists", async () => { - selectResult.current = [{ value: "0.1.0" }]; + it("returns the ensNodeSchema", () => { + const ensDbReader = createEnsDbReader(); + expect(ensDbReader.ensNodeSchema).toBeDefined(); + }); + }); - await expect(createEnsDbReader().getEnsDbVersion()).resolves.toBe("0.1.0"); + describe("buildEnsDbPublicConfig", () => { + it("returns version info with the postgresql version", async () => { + executeMock.mockResolvedValueOnce({ + rows: [ + { + version: "PostgreSQL 17.4 (Ubuntu 17.4-0ubuntu0.22.04.1) on x86_64-pc-linux-gnu", + }, + ], + }); + + const result = await createEnsDbReader().buildEnsDbPublicConfig(); + + expect(result).toStrictEqual({ + versionInfo: { + postgresql: "17.4", + }, + } satisfies EnsDbPublicConfig); + expect(executeMock).toHaveBeenCalledWith("SELECT version();"); }); - // This scenario should be impossible due to the primary key constraint on - // the ('ensIndexerSchemaName', 'key') columns of the 'ensnode_metadata' table. - it("throws when multiple records exist", async () => { - selectResult.current = [{ value: "0.1.0" }, { value: "0.1.1" }]; + it("throws when execute returns no rows", async () => { + executeMock.mockResolvedValueOnce({ rows: [] }); + + await expect(createEnsDbReader().buildEnsDbPublicConfig()).rejects.toThrow( + /Failed to get PostgreSQL version/, + ); + }); + + it("throws when execute returns an invalid version string", async () => { + executeMock.mockResolvedValueOnce({ + rows: [{ version: "invalid version string" }], + }); - await expect(createEnsDbReader().getEnsDbVersion()).rejects.toThrowError(/ensdb_version/i); + await expect(createEnsDbReader().buildEnsDbPublicConfig()).rejects.toThrow( + /Failed to get PostgreSQL version/, + ); + }); + + it("propagates errors from execute", async () => { + executeMock.mockRejectedValueOnce(new Error("Connection refused")); + + await expect(createEnsDbReader().buildEnsDbPublicConfig()).rejects.toThrow( + "Connection refused", + ); }); }); - describe("getEnsIndexerPublicConfig", () => { - it("returns undefined when no record exists", async () => { - await expect(createEnsDbReader().getEnsIndexerPublicConfig()).resolves.toBeUndefined(); + describe("isHealthy", () => { + it("returns true when execute succeeds", async () => { + executeMock.mockResolvedValueOnce({ rows: [] }); + + const result = await createEnsDbReader().isHealthy(); + + expect(result).toBe(true); }); - it("deserializes the stored config", async () => { - const serializedConfig = serializeEnsIndexerPublicConfig(ensDbClientMock.publicConfig); - selectResult.current = [{ value: serializedConfig }]; + it("returns false when execute fails", async () => { + executeMock.mockRejectedValueOnce(new Error("Connection refused")); - await expect(createEnsDbReader().getEnsIndexerPublicConfig()).resolves.toStrictEqual( + const result = await createEnsDbReader().isHealthy(); + + expect(result).toBe(false); + }); + }); + + describe("isReady", () => { + it("returns true when healthy and indexing metadata context is initialized", async () => { + executeMock.mockResolvedValueOnce({ rows: [] }); + + const indexingStatus = deserializeCrossChainIndexingStatusSnapshot( + ensDbClientMock.serializedSnapshot, + ); + const ensDbPublicConfig: EnsDbPublicConfig = { + versionInfo: { postgresql: "17.4" }, + }; + const ensRainbowPublicConfig = { + serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, + versionInfo: { ensRainbow: "1.9.0" }, + }; + const stackInfo = buildEnsIndexerStackInfo( + ensDbPublicConfig, ensDbClientMock.publicConfig, + ensRainbowPublicConfig, ); + const context = buildIndexingMetadataContextInitialized(indexingStatus, stackInfo); + const serialized = serializeIndexingMetadataContext(context); + selectResult.current = [{ value: serialized }]; + + const result = await createEnsDbReader().isReady(); + + expect(result).toBe(true); + }); + + it("returns false when healthy but indexing metadata context is uninitialized", async () => { + executeMock.mockResolvedValueOnce({ rows: [] }); + selectResult.current = []; + + const result = await createEnsDbReader().isReady(); + + expect(result).toBe(false); + }); + + it("returns false when not healthy", async () => { + executeMock.mockRejectedValueOnce(new Error("Connection refused")); + + const result = await createEnsDbReader().isReady(); + + expect(result).toBe(false); }); }); - describe("getIndexingStatusSnapshot", () => { - it("deserializes the stored indexing status snapshot", async () => { - selectResult.current = [{ value: ensDbClientMock.serializedSnapshot }]; + describe("getIndexingMetadataContext", () => { + it("returns an uninitialized context when no record exists", async () => { + const ensDbReader = createEnsDbReader(); + const { ensNodeSchema } = ensDbReader; + + const result = await ensDbReader.getIndexingMetadataContext(); + + expect(result).toStrictEqual(buildIndexingMetadataContextUninitialized()); + expect(selectMock).toHaveBeenCalledTimes(1); + expect(fromMock).toHaveBeenCalledWith(ensNodeSchema.metadata); + expect(whereMock).toHaveBeenCalled(); + }); - const expected = deserializeCrossChainIndexingStatusSnapshot( + it("returns the deserialized initialized context when one record exists", async () => { + const indexingStatus = deserializeCrossChainIndexingStatusSnapshot( ensDbClientMock.serializedSnapshot, ); + const ensDbPublicConfig: EnsDbPublicConfig = { + versionInfo: { postgresql: "17.4" }, + }; + const ensRainbowPublicConfig = { + serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, + versionInfo: { ensRainbow: "1.9.0" }, + }; + const stackInfo = buildEnsIndexerStackInfo( + ensDbPublicConfig, + ensDbClientMock.publicConfig, + ensRainbowPublicConfig, + ); + const context = buildIndexingMetadataContextInitialized(indexingStatus, stackInfo); + const serialized = serializeIndexingMetadataContext(context); + + selectResult.current = [{ value: serialized }]; + + const result = await createEnsDbReader().getIndexingMetadataContext(); + + expect(result).toStrictEqual(context); + }); + + // This scenario should be impossible due to the primary key constraint on + // the ('ensIndexerSchemaName', 'key') columns of the 'ensnode_metadata' table. + it("throws when multiple records exist", async () => { + selectResult.current = [{ value: "value1" }, { value: "value2" }]; - await expect(createEnsDbReader().getIndexingStatusSnapshot()).resolves.toStrictEqual( - expected, + await expect(createEnsDbReader().getIndexingMetadataContext()).rejects.toThrow( + /There must be exactly one ENSNodeMetadata record/, ); }); }); diff --git a/packages/ensdb-sdk/src/client/ensdb-reader.ts b/packages/ensdb-sdk/src/client/ensdb-reader.ts index 49d61d3ee1..22924bba4f 100644 --- a/packages/ensdb-sdk/src/client/ensdb-reader.ts +++ b/packages/ensdb-sdk/src/client/ensdb-reader.ts @@ -1,12 +1,19 @@ import { and, eq } from "drizzle-orm/sql"; import { - type CrossChainIndexingStatusSnapshot, - deserializeCrossChainIndexingStatusSnapshot, - deserializeEnsIndexerPublicConfig, + buildIndexingMetadataContextUninitialized, + deserializeIndexingMetadataContext, type EnsDbPublicConfig, type EnsDbVersionInfo, - type EnsIndexerPublicConfig, + type IndexingMetadataContext, + IndexingMetadataContextStatusCodes, +} from "@ensnode/ensnode-sdk"; + +export { + type IndexingMetadataContext, + type IndexingMetadataContextInitialized, + IndexingMetadataContextStatusCodes, + type IndexingMetadataContextUninitialized, } from "@ensnode/ensnode-sdk"; import { @@ -20,9 +27,7 @@ import { parsePgVersionInfo } from "../lib/parse-pg-version-info"; import { EnsNodeMetadataKeys } from "./ensnode-metadata"; import type { SerializedEnsNodeMetadata, - SerializedEnsNodeMetadataEnsDbVersion, - SerializedEnsNodeMetadataEnsIndexerIndexingStatus, - SerializedEnsNodeMetadataEnsIndexerPublicConfig, + SerializedEnsNodeMetadataIndexingMetadataContext, } from "./serialize/ensnode-metadata"; /** @@ -130,33 +135,33 @@ export class EnsDbReader< } /** - * Get ENSDb Version - * - * @returns the existing record, or `undefined`. + * Check if the ENSDb instance is healthy by running a simple query + * against it. */ - async getEnsDbVersion(): Promise { - const record = await this.getEnsNodeMetadata({ - key: EnsNodeMetadataKeys.EnsDbVersion, - }); - - return record; + async isHealthy(): Promise { + try { + await this.ensDb.execute("SELECT 1;"); + return true; + } catch { + return false; + } } /** - * Get ENSIndexer Public Config - * - * @returns the existing record, or `undefined`. + * Check if the ENSDb instance is ready by verifying that it is + * healthy and the {@link IndexingMetadataContext} has been initialized for + * the ENSIndexer Schema used by this ENSDbReader instance. */ - async getEnsIndexerPublicConfig(): Promise { - const record = await this.getEnsNodeMetadata({ - key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, - }); + async isReady(): Promise { + const isHealthy = await this.isHealthy(); - if (!record) { - return undefined; + if (!isHealthy) { + return false; } - return deserializeEnsIndexerPublicConfig(record); + const indexingMetadataContext = await this.getIndexingMetadataContext(); + + return indexingMetadataContext.statusCode === IndexingMetadataContextStatusCodes.Initialized; } /** @@ -171,22 +176,20 @@ export class EnsDbReader< } /** - * Get Indexing Status Snapshot + * Get Indexing Metadata Context * - * @returns the existing record, or `undefined`. + * @returns the initialized record, or a default uninitialized one if no record exists in ENSDb. */ - async getIndexingStatusSnapshot(): Promise { - const record = await this.getEnsNodeMetadata( - { - key: EnsNodeMetadataKeys.EnsIndexerIndexingStatus, - }, - ); + async getIndexingMetadataContext(): Promise { + const record = await this.getEnsNodeMetadata({ + key: EnsNodeMetadataKeys.IndexingMetadataContext, + }); if (!record) { - return undefined; + return buildIndexingMetadataContextUninitialized(); } - return deserializeCrossChainIndexingStatusSnapshot(record); + return deserializeIndexingMetadataContext(record); } /** diff --git a/packages/ensdb-sdk/src/client/ensdb-writer.test.ts b/packages/ensdb-sdk/src/client/ensdb-writer.test.ts index 0ec99aaa35..27e3dee565 100644 --- a/packages/ensdb-sdk/src/client/ensdb-writer.test.ts +++ b/packages/ensdb-sdk/src/client/ensdb-writer.test.ts @@ -2,9 +2,10 @@ import { migrate } from "drizzle-orm/node-postgres/migrator"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { + buildEnsIndexerStackInfo, + buildIndexingMetadataContextInitialized, deserializeCrossChainIndexingStatusSnapshot, - serializeCrossChainIndexingStatusSnapshot, - serializeEnsIndexerPublicConfig, + serializeIndexingMetadataContext, } from "@ensnode/ensnode-sdk"; import * as ensDbClientMock from "./ensdb-client.mock"; @@ -43,54 +44,41 @@ describe("EnsDbWriter", () => { vi.mocked(migrate).mockClear(); }); - describe("upsertEnsDbVersion", () => { - it("writes the database version metadata", async () => { - const ensDbClient = createEnsDbWriter(); - const { ensNodeSchema } = ensDbClient; + describe("upsertIndexingMetadataContext", () => { + it("serializes and writes the indexing metadata context", async () => { + const ensDbWriter = createEnsDbWriter(); + const { ensNodeSchema } = ensDbWriter; - await ensDbClient.upsertEnsDbVersion("0.2.0"); - - expect(insertMock).toHaveBeenCalledWith(ensNodeSchema.metadata); - expect(valuesMock).toHaveBeenCalledWith({ - ensIndexerSchemaName: ensDbClientMock.ensIndexerSchemaName, - key: EnsNodeMetadataKeys.EnsDbVersion, - value: "0.2.0", - }); - expect(onConflictDoUpdateMock).toHaveBeenCalledWith({ - target: [ensNodeSchema.metadata.ensIndexerSchemaName, ensNodeSchema.metadata.key], - set: { value: "0.2.0" }, - }); - }); - }); - - describe("upsertEnsIndexerPublicConfig", () => { - it("serializes and writes the public config", async () => { - const expectedValue = serializeEnsIndexerPublicConfig(ensDbClientMock.publicConfig); - - await createEnsDbWriter().upsertEnsIndexerPublicConfig(ensDbClientMock.publicConfig); - - expect(valuesMock).toHaveBeenCalledWith({ - ensIndexerSchemaName: ensDbClientMock.ensIndexerSchemaName, - key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, - value: expectedValue, - }); - }); - }); - - describe("upsertIndexingStatusSnapshot", () => { - it("serializes and writes the indexing status snapshot", async () => { - const snapshot = deserializeCrossChainIndexingStatusSnapshot( + const indexingStatus = deserializeCrossChainIndexingStatusSnapshot( ensDbClientMock.serializedSnapshot, ); - const expectedValue = serializeCrossChainIndexingStatusSnapshot(snapshot); + const ensDbPublicConfig = { + versionInfo: { postgresql: "17.4" }, + }; + const ensRainbowPublicConfig = { + serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, + versionInfo: { ensRainbow: "1.9.0" }, + }; + const stackInfo = buildEnsIndexerStackInfo( + ensDbPublicConfig, + ensDbClientMock.publicConfig, + ensRainbowPublicConfig, + ); + const context = buildIndexingMetadataContextInitialized(indexingStatus, stackInfo); + const expectedValue = serializeIndexingMetadataContext(context); - await createEnsDbWriter().upsertIndexingStatusSnapshot(snapshot); + await ensDbWriter.upsertIndexingMetadataContext(context); + expect(insertMock).toHaveBeenCalledWith(ensNodeSchema.metadata); expect(valuesMock).toHaveBeenCalledWith({ ensIndexerSchemaName: ensDbClientMock.ensIndexerSchemaName, - key: EnsNodeMetadataKeys.EnsIndexerIndexingStatus, + key: EnsNodeMetadataKeys.IndexingMetadataContext, value: expectedValue, }); + expect(onConflictDoUpdateMock).toHaveBeenCalledWith({ + target: [ensNodeSchema.metadata.ensIndexerSchemaName, ensNodeSchema.metadata.key], + set: { value: expectedValue }, + }); }); }); diff --git a/packages/ensdb-sdk/src/client/ensdb-writer.ts b/packages/ensdb-sdk/src/client/ensdb-writer.ts index c7841ed32d..022308fcb8 100644 --- a/packages/ensdb-sdk/src/client/ensdb-writer.ts +++ b/packages/ensdb-sdk/src/client/ensdb-writer.ts @@ -2,10 +2,8 @@ import { sql } from "drizzle-orm"; import { migrate } from "drizzle-orm/node-postgres/migrator"; import { - type CrossChainIndexingStatusSnapshot, - type EnsIndexerPublicConfig, - serializeCrossChainIndexingStatusSnapshot, - serializeEnsIndexerPublicConfig, + type IndexingMetadataContextInitialized, + serializeIndexingMetadataContext, } from "@ensnode/ensnode-sdk"; import { advisoryLockId } from "../lib/advisory-lock-id"; @@ -59,42 +57,16 @@ export class EnsDbWriter extends EnsDbReader { } /** - * Upsert ENSDb Version + * Upsert Indexing Metadata Context Initialized * * @throws when upsert operation failed. */ - async upsertEnsDbVersion(ensDbVersion: string): Promise { - await this.upsertEnsNodeMetadata({ - key: EnsNodeMetadataKeys.EnsDbVersion, - value: ensDbVersion, - }); - } - - /** - * Upsert ENSIndexer Public Config - * - * @throws when upsert operation failed. - */ - async upsertEnsIndexerPublicConfig( - ensIndexerPublicConfig: EnsIndexerPublicConfig, - ): Promise { - await this.upsertEnsNodeMetadata({ - key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, - value: serializeEnsIndexerPublicConfig(ensIndexerPublicConfig), - }); - } - - /** - * Upsert Indexing Status Snapshot - * - * @throws when upsert operation failed. - */ - async upsertIndexingStatusSnapshot( - indexingStatus: CrossChainIndexingStatusSnapshot, + async upsertIndexingMetadataContext( + indexingMetadataContext: IndexingMetadataContextInitialized, ): Promise { await this.upsertEnsNodeMetadata({ - key: EnsNodeMetadataKeys.EnsIndexerIndexingStatus, - value: serializeCrossChainIndexingStatusSnapshot(indexingStatus), + key: EnsNodeMetadataKeys.IndexingMetadataContext, + value: serializeIndexingMetadataContext(indexingMetadataContext), }); } diff --git a/packages/ensdb-sdk/src/client/ensnode-metadata.ts b/packages/ensdb-sdk/src/client/ensnode-metadata.ts index bdb35c4069..9b0a1a2d50 100644 --- a/packages/ensdb-sdk/src/client/ensnode-metadata.ts +++ b/packages/ensdb-sdk/src/client/ensnode-metadata.ts @@ -1,40 +1,31 @@ -import type { - CrossChainIndexingStatusSnapshot, - EnsIndexerPublicConfig, -} from "@ensnode/ensnode-sdk"; +import type { IndexingMetadataContextInitialized } from "@ensnode/ensnode-sdk"; /** * Keys used to distinguish records in `ensnode_metadata` table in the ENSDb. */ export const EnsNodeMetadataKeys = { - EnsDbVersion: "ensdb_version", - EnsIndexerPublicConfig: "ensindexer_public_config", - EnsIndexerIndexingStatus: "ensindexer_indexing_status", + IndexingMetadataContext: "indexing_metadata_context", } as const; export type EnsNodeMetadataKey = (typeof EnsNodeMetadataKeys)[keyof typeof EnsNodeMetadataKeys]; -export interface EnsNodeMetadataEnsDbVersion { - key: typeof EnsNodeMetadataKeys.EnsDbVersion; - value: string; -} - -export interface EnsNodeMetadataEnsIndexerPublicConfig { - key: typeof EnsNodeMetadataKeys.EnsIndexerPublicConfig; - value: EnsIndexerPublicConfig; -} - -export interface EnsNodeMetadataEnsIndexerIndexingStatus { - key: typeof EnsNodeMetadataKeys.EnsIndexerIndexingStatus; - value: CrossChainIndexingStatusSnapshot; +/** + * ENSNode Metadata record for Indexing Metadata Context + * + * This record is used to store the Indexing Metadata Context in + * ENSNode Metadata table for each ENSIndexer instance. + */ +export interface EnsNodeMetadataIndexingMetadataContext { + key: typeof EnsNodeMetadataKeys.IndexingMetadataContext; + value: IndexingMetadataContextInitialized; } /** * ENSNode Metadata * - * Union type gathering all variants of ENSNode Metadata. + * Type alias for ENSNode Metadata records, + * currently only includes the record for Indexing Metadata Context, + * but can be extended in the future to include more types of + * ENSNode Metadata records as needed. */ -export type EnsNodeMetadata = - | EnsNodeMetadataEnsDbVersion - | EnsNodeMetadataEnsIndexerPublicConfig - | EnsNodeMetadataEnsIndexerIndexingStatus; +export type EnsNodeMetadata = EnsNodeMetadataIndexingMetadataContext; diff --git a/packages/ensdb-sdk/src/client/serialize/ensnode-metadata.ts b/packages/ensdb-sdk/src/client/serialize/ensnode-metadata.ts index cae7fcdd34..aeacf63b4d 100644 --- a/packages/ensdb-sdk/src/client/serialize/ensnode-metadata.ts +++ b/packages/ensdb-sdk/src/client/serialize/ensnode-metadata.ts @@ -1,41 +1,13 @@ -import type { - SerializedCrossChainIndexingStatusSnapshot, - SerializedEnsIndexerPublicConfig, -} from "@ensnode/ensnode-sdk"; +import type { SerializedIndexingMetadataContextInitialized } from "@ensnode/ensnode-sdk"; -import type { - EnsNodeMetadata, - EnsNodeMetadataEnsDbVersion, - EnsNodeMetadataEnsIndexerIndexingStatus, - EnsNodeMetadataEnsIndexerPublicConfig, - EnsNodeMetadataKeys, -} from "../ensnode-metadata"; +import type { EnsNodeMetadata, EnsNodeMetadataKeys } from "../ensnode-metadata"; -/** - * Serialized representation of {@link EnsNodeMetadataEnsDbVersion}. - */ -export type SerializedEnsNodeMetadataEnsDbVersion = EnsNodeMetadataEnsDbVersion; - -/** - * Serialized representation of {@link EnsNodeMetadataEnsIndexerPublicConfig}. - */ -export interface SerializedEnsNodeMetadataEnsIndexerPublicConfig { - key: typeof EnsNodeMetadataKeys.EnsIndexerPublicConfig; - value: SerializedEnsIndexerPublicConfig; -} - -/** - * Serialized representation of {@link EnsNodeMetadataEnsIndexerIndexingStatus}. - */ -export interface SerializedEnsNodeMetadataEnsIndexerIndexingStatus { - key: typeof EnsNodeMetadataKeys.EnsIndexerIndexingStatus; - value: SerializedCrossChainIndexingStatusSnapshot; +export interface SerializedEnsNodeMetadataIndexingMetadataContext { + key: typeof EnsNodeMetadataKeys.IndexingMetadataContext; + value: SerializedIndexingMetadataContextInitialized; } /** * Serialized representation of {@link EnsNodeMetadata} */ -export type SerializedEnsNodeMetadata = - | SerializedEnsNodeMetadataEnsDbVersion - | SerializedEnsNodeMetadataEnsIndexerPublicConfig - | SerializedEnsNodeMetadataEnsIndexerIndexingStatus; +export type SerializedEnsNodeMetadata = SerializedEnsNodeMetadataIndexingMetadataContext; diff --git a/packages/ensnode-sdk/src/ensnode/index.ts b/packages/ensnode-sdk/src/ensnode/index.ts index 227ed19cdb..ece02b5ce5 100644 --- a/packages/ensnode-sdk/src/ensnode/index.ts +++ b/packages/ensnode-sdk/src/ensnode/index.ts @@ -5,3 +5,4 @@ export { } from "./client"; export * from "./client-error"; export * from "./deployments"; +export * from "./metadata"; diff --git a/packages/ensnode-sdk/src/ensnode/metadata/deserialize/indexing-metadata-context.ts b/packages/ensnode-sdk/src/ensnode/metadata/deserialize/indexing-metadata-context.ts new file mode 100644 index 0000000000..2ada8c6bec --- /dev/null +++ b/packages/ensnode-sdk/src/ensnode/metadata/deserialize/indexing-metadata-context.ts @@ -0,0 +1,73 @@ +import { prettifyError } from "zod/v4"; + +import { buildUnvalidatedCrossChainIndexingStatusSnapshot } from "../../../indexing-status"; +import type { Unvalidated } from "../../../shared/types"; +import { buildUnvalidatedEnsIndexerStackInfo } from "../../../stack-info"; +import { + type IndexingMetadataContext, + type IndexingMetadataContextInitialized, + IndexingMetadataContextStatusCodes, +} from "../indexing-metadata-context"; +import type { + SerializedIndexingMetadataContext, + SerializedIndexingMetadataContextInitialized, +} from "../serialize/indexing-metadata-context"; +import { + makeIndexingMetadataContextSchema, + makeSerializedIndexingMetadataContextSchema, +} from "../zod-schemas/indexing-metadata-context"; + +/** + * Builds an unvalidated {@link IndexingMetadataContextInitialized} object. + */ +function buildUnvalidatedIndexingMetadataContextInitialized( + serializedIndexingMetadataContext: SerializedIndexingMetadataContextInitialized, +): Unvalidated { + return { + statusCode: serializedIndexingMetadataContext.statusCode, + indexingStatus: buildUnvalidatedCrossChainIndexingStatusSnapshot( + serializedIndexingMetadataContext.indexingStatus, + ), + stackInfo: buildUnvalidatedEnsIndexerStackInfo(serializedIndexingMetadataContext.stackInfo), + }; +} + +/** + * Builds an unvalidated {@link IndexingMetadataContext} object to be + * validated with {@link makeIndexingMetadataContextSchema}. + * + * @param serializedIndexingMetadataContext - The serialized indexing metadata context to build from. + */ +function buildUnvalidatedIndexingMetadataContext( + serializedIndexingMetadataContext: SerializedIndexingMetadataContext, +): Unvalidated { + switch (serializedIndexingMetadataContext.statusCode) { + case IndexingMetadataContextStatusCodes.Uninitialized: + return serializedIndexingMetadataContext; + + case IndexingMetadataContextStatusCodes.Initialized: + return buildUnvalidatedIndexingMetadataContextInitialized(serializedIndexingMetadataContext); + } +} + +/** + * Deserialize a serialized {@link IndexingMetadataContext} object. + */ +export function deserializeIndexingMetadataContext( + serializedIndexingMetadataContext: Unvalidated, + valueLabel?: string, +): IndexingMetadataContext { + const label = valueLabel ?? "IndexingMetadataContext"; + + const parsed = makeSerializedIndexingMetadataContextSchema(label) + .transform(buildUnvalidatedIndexingMetadataContext) + .pipe(makeIndexingMetadataContextSchema(label)) + .safeParse(serializedIndexingMetadataContext); + + if (parsed.error) { + throw new Error( + `Cannot deserialize IndexingMetadataContext:\n${prettifyError(parsed.error)}\n`, + ); + } + return parsed.data; +} diff --git a/packages/ensnode-sdk/src/ensnode/metadata/index.ts b/packages/ensnode-sdk/src/ensnode/metadata/index.ts new file mode 100644 index 0000000000..483ae202d8 --- /dev/null +++ b/packages/ensnode-sdk/src/ensnode/metadata/index.ts @@ -0,0 +1,4 @@ +export * from "./deserialize/indexing-metadata-context"; +export * from "./indexing-metadata-context"; +export * from "./serialize/indexing-metadata-context"; +export * from "./validate/indexing-metadata-context"; diff --git a/packages/ensnode-sdk/src/ensnode/metadata/indexing-metadata-context.test.ts b/packages/ensnode-sdk/src/ensnode/metadata/indexing-metadata-context.test.ts new file mode 100644 index 0000000000..3fbf0fe191 --- /dev/null +++ b/packages/ensnode-sdk/src/ensnode/metadata/indexing-metadata-context.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from "vitest"; + +import { ChainIndexingStatusIds } from "../../indexing-status/chain-indexing-status-snapshot"; +import { buildCrossChainIndexingStatusSnapshotOmnichain } from "../../indexing-status/cross-chain-indexing-status-snapshot"; +import { OmnichainIndexingStatusIds } from "../../indexing-status/omnichain-indexing-status-snapshot"; +import { RangeTypeIds } from "../../shared/blockrange"; +import { buildEnsIndexerStackInfo } from "../../stack-info/ensindexer-stack-info"; +import { deserializeIndexingMetadataContext } from "./deserialize/indexing-metadata-context"; +import { + buildIndexingMetadataContextInitialized, + buildIndexingMetadataContextUninitialized, + IndexingMetadataContextStatusCodes, +} from "./indexing-metadata-context"; +import { serializeIndexingMetadataContext } from "./serialize/indexing-metadata-context"; +import { validateIndexingMetadataContextInitialized } from "./validate/indexing-metadata-context"; + +function makeMinimalIndexingStatus() { + const earlierBlock = { timestamp: 1672531199, number: 1024 }; + const laterBlock = { timestamp: 1672531200, number: 1025 }; + const snapshotTime = laterBlock.timestamp; + + return buildCrossChainIndexingStatusSnapshotOmnichain( + { + omnichainStatus: OmnichainIndexingStatusIds.Following, + chains: new Map([ + [ + 1, + { + chainStatus: ChainIndexingStatusIds.Following, + config: { rangeType: RangeTypeIds.LeftBounded, startBlock: earlierBlock }, + latestIndexedBlock: earlierBlock, + latestKnownBlock: laterBlock, + }, + ], + ]), + omnichainIndexingCursor: earlierBlock.timestamp, + }, + snapshotTime, + ); +} + +function makeMinimalStackInfo() { + return buildEnsIndexerStackInfo( + { versionInfo: { postgresql: "17.4" } }, + { + namespace: "mainnet", + clientLabelSet: { labelSetId: "subgraph", labelSetVersion: 0 }, + ensIndexerSchemaName: "test_schema", + ensRainbowPublicConfig: { + serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, + versionInfo: { ensRainbow: "1.0.0" }, + }, + indexedChainIds: new Set([1]), + isSubgraphCompatible: true, + plugins: ["subgraph"], + versionInfo: { + ponder: "0.11.25", + ensDb: "0.32.0", + ensIndexer: "0.32.0", + ensNormalize: "1.11.1", + }, + }, + { + serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 }, + versionInfo: { ensRainbow: "1.0.0" }, + }, + ); +} + +describe("buildIndexingMetadataContextUninitialized", () => { + it('returns an object with statusCode "uninitialized"', () => { + const result = buildIndexingMetadataContextUninitialized(); + + expect(result).toStrictEqual({ + statusCode: IndexingMetadataContextStatusCodes.Uninitialized, + }); + }); +}); + +describe("buildIndexingMetadataContextInitialized", () => { + it("constructs a valid initialized context from valid inputs", () => { + const indexingStatus = makeMinimalIndexingStatus(); + const stackInfo = makeMinimalStackInfo(); + + const result = buildIndexingMetadataContextInitialized(indexingStatus, stackInfo); + + expect(result.statusCode).toBe(IndexingMetadataContextStatusCodes.Initialized); + expect(result.indexingStatus).toStrictEqual(indexingStatus); + expect(result.stackInfo).toStrictEqual(stackInfo); + }); + + it("throws when indexingStatus is invalid", () => { + const stackInfo = makeMinimalStackInfo(); + + expect(() => buildIndexingMetadataContextInitialized({} as any, stackInfo)).toThrow(); + }); + + it("throws when stackInfo is invalid", () => { + const indexingStatus = makeMinimalIndexingStatus(); + + expect(() => buildIndexingMetadataContextInitialized(indexingStatus, {} as any)).toThrow(); + }); +}); + +describe("serializeIndexingMetadataContext / deserializeIndexingMetadataContext roundtrip", () => { + it("roundtrips an uninitialized context", () => { + const context = buildIndexingMetadataContextUninitialized(); + const serialized = serializeIndexingMetadataContext(context); + const deserialized = deserializeIndexingMetadataContext(serialized); + + expect(deserialized).toStrictEqual(context); + }); + + it("roundtrips an initialized context", () => { + const context = buildIndexingMetadataContextInitialized( + makeMinimalIndexingStatus(), + makeMinimalStackInfo(), + ); + const serialized = serializeIndexingMetadataContext(context); + const deserialized = deserializeIndexingMetadataContext(serialized); + + expect(deserialized).toStrictEqual(context); + }); +}); + +describe("deserializeIndexingMetadataContext", () => { + it("rejects malformed serialized data (missing statusCode)", () => { + expect(() => deserializeIndexingMetadataContext({} as any)).toThrow(); + }); + + it("rejects serialized data with unrecognized statusCode", () => { + expect(() => deserializeIndexingMetadataContext({ statusCode: "bogus" } as any)).toThrow(); + }); + + it("rejects non-object input", () => { + expect(() => deserializeIndexingMetadataContext(null as any)).toThrow(); + expect(() => deserializeIndexingMetadataContext("not an object" as any)).toThrow(); + }); +}); + +describe("validateIndexingMetadataContextInitialized", () => { + it("rejects an uninitialized statusCode", () => { + expect(() => + validateIndexingMetadataContextInitialized({ + statusCode: IndexingMetadataContextStatusCodes.Uninitialized, + } as any), + ).toThrow(); + }); +}); diff --git a/packages/ensnode-sdk/src/ensnode/metadata/indexing-metadata-context.ts b/packages/ensnode-sdk/src/ensnode/metadata/indexing-metadata-context.ts new file mode 100644 index 0000000000..457a9cd2ce --- /dev/null +++ b/packages/ensnode-sdk/src/ensnode/metadata/indexing-metadata-context.ts @@ -0,0 +1,72 @@ +import type { CrossChainIndexingStatusSnapshot } from "../../indexing-status"; +import type { EnsIndexerStackInfo } from "../../stack-info"; +import { validateIndexingMetadataContextInitialized } from "./validate/indexing-metadata-context"; + +/** + * A status code for an indexing metadata context + */ +export const IndexingMetadataContextStatusCodes = { + /** + * Represents that no indexing metadata context has been initialized + * for the ENSIndexer Schema Name in the ENSNode Metadata table in ENSDb. + */ + Uninitialized: "uninitialized", + + /** + * Represents that the indexing metadata context has been initialized + * for the ENSIndexer Schema Name in the ENSNode Metadata table in ENSDb. + */ + Initialized: "initialized", +} as const; + +/** + * The derived string union of possible {@link IndexingMetadataContextStatusCodes}. + */ +export type IndexingMetadataContextStatusCode = + (typeof IndexingMetadataContextStatusCodes)[keyof typeof IndexingMetadataContextStatusCodes]; + +export interface IndexingMetadataContextUninitialized { + statusCode: typeof IndexingMetadataContextStatusCodes.Uninitialized; +} + +export interface IndexingMetadataContextInitialized { + statusCode: typeof IndexingMetadataContextStatusCodes.Initialized; + indexingStatus: CrossChainIndexingStatusSnapshot; + stackInfo: EnsIndexerStackInfo; +} + +/** + * Indexing Metadata Context + * + * Use the {@link IndexingMetadataContext.statusCode} field to determine + * the specific type interpretation at runtime. + */ +export type IndexingMetadataContext = + | IndexingMetadataContextUninitialized + | IndexingMetadataContextInitialized; + +/** + * Build an {@link IndexingMetadataContextUninitialized} object. + */ +export function buildIndexingMetadataContextUninitialized(): IndexingMetadataContextUninitialized { + return { + statusCode: IndexingMetadataContextStatusCodes.Uninitialized, + }; +} + +/** + * Build an {@link IndexingMetadataContextInitialized} object. + * + * @throws Error if the provided parameters do not satisfy the validation + * criteria for an {@link IndexingMetadataContextInitialized} object. + */ +export function buildIndexingMetadataContextInitialized( + indexingStatus: CrossChainIndexingStatusSnapshot, + stackInfo: EnsIndexerStackInfo, +): IndexingMetadataContextInitialized { + return validateIndexingMetadataContextInitialized({ + statusCode: IndexingMetadataContextStatusCodes.Initialized, + indexingStatus, + stackInfo, + }); +} diff --git a/packages/ensnode-sdk/src/ensnode/metadata/serialize/indexing-metadata-context.ts b/packages/ensnode-sdk/src/ensnode/metadata/serialize/indexing-metadata-context.ts new file mode 100644 index 0000000000..cde334e7a9 --- /dev/null +++ b/packages/ensnode-sdk/src/ensnode/metadata/serialize/indexing-metadata-context.ts @@ -0,0 +1,66 @@ +import { + type SerializedCrossChainIndexingStatusSnapshot, + serializeCrossChainIndexingStatusSnapshot, +} from "../../../indexing-status/serialize/cross-chain-indexing-status-snapshot"; +import { + type SerializedEnsIndexerStackInfo, + serializeEnsIndexerStackInfo, +} from "../../../stack-info/serialize/ensindexer-stack-info"; +import { + type IndexingMetadataContext, + type IndexingMetadataContextInitialized, + IndexingMetadataContextStatusCodes, + type IndexingMetadataContextUninitialized, +} from "../indexing-metadata-context"; + +/** + * Serialized representation of an {@link IndexingMetadataContextUninitialized}. + */ +export type SerializedIndexingMetadataContextUninitialized = IndexingMetadataContextUninitialized; + +/** + * Serialized representation of an {@link IndexingMetadataContextInitialized}. + */ +export interface SerializedIndexingMetadataContextInitialized + extends Omit { + indexingStatus: SerializedCrossChainIndexingStatusSnapshot; + stackInfo: SerializedEnsIndexerStackInfo; +} + +/** + * Serialized representation of an {@link IndexingMetadataContext}. + * + * Use the {@link SerializedIndexingMetadataContext.statusCode} field to + * determine the specific type interpretation at runtime. + */ +export type SerializedIndexingMetadataContext = + | SerializedIndexingMetadataContextUninitialized + | SerializedIndexingMetadataContextInitialized; + +export function serializeIndexingMetadataContextInitialized( + context: IndexingMetadataContextInitialized, +): SerializedIndexingMetadataContextInitialized { + const { statusCode, indexingStatus, stackInfo } = context; + return { + statusCode, + indexingStatus: serializeCrossChainIndexingStatusSnapshot(indexingStatus), + stackInfo: serializeEnsIndexerStackInfo(stackInfo), + }; +} + +export function serializeIndexingMetadataContext( + context: IndexingMetadataContextUninitialized, +): SerializedIndexingMetadataContextUninitialized; +export function serializeIndexingMetadataContext( + context: IndexingMetadataContextInitialized, +): SerializedIndexingMetadataContextInitialized; +export function serializeIndexingMetadataContext( + context: IndexingMetadataContext, +): SerializedIndexingMetadataContext { + switch (context.statusCode) { + case IndexingMetadataContextStatusCodes.Uninitialized: + return context; + case IndexingMetadataContextStatusCodes.Initialized: + return serializeIndexingMetadataContextInitialized(context); + } +} diff --git a/packages/ensnode-sdk/src/ensnode/metadata/validate/indexing-metadata-context.ts b/packages/ensnode-sdk/src/ensnode/metadata/validate/indexing-metadata-context.ts new file mode 100644 index 0000000000..290eaa1e7d --- /dev/null +++ b/packages/ensnode-sdk/src/ensnode/metadata/validate/indexing-metadata-context.ts @@ -0,0 +1,23 @@ +import { prettifyError } from "zod/v4"; + +import type { Unvalidated } from "../../../shared/types"; +import type { IndexingMetadataContextInitialized } from "../indexing-metadata-context"; +import { makeIndexingMetadataContextInitializedSchema } from "../zod-schemas/indexing-metadata-context"; + +/** + * Validate a maybe {@link IndexingMetadataContextInitialized} object. + */ +export function validateIndexingMetadataContextInitialized( + maybeIndexingMetadataContext: Unvalidated, + valueLabel?: string, +): IndexingMetadataContextInitialized { + const label = valueLabel ?? "IndexingMetadataContextInitialized"; + const result = makeIndexingMetadataContextInitializedSchema(label).safeParse( + maybeIndexingMetadataContext, + ); + + if (result.error) { + throw new Error(`Cannot validate ${label}:\n${prettifyError(result.error)}\n`); + } + return result.data; +} diff --git a/packages/ensnode-sdk/src/ensnode/metadata/zod-schemas/indexing-metadata-context.ts b/packages/ensnode-sdk/src/ensnode/metadata/zod-schemas/indexing-metadata-context.ts new file mode 100644 index 0000000000..10291b1627 --- /dev/null +++ b/packages/ensnode-sdk/src/ensnode/metadata/zod-schemas/indexing-metadata-context.ts @@ -0,0 +1,66 @@ +import { z } from "zod/v4"; + +import { + makeCrossChainIndexingStatusSnapshotSchema, + makeSerializedCrossChainIndexingStatusSnapshotSchema, +} from "../../../indexing-status/zod-schema/cross-chain-indexing-status-snapshot"; +import { + makeEnsIndexerStackInfoSchema, + makeSerializedEnsIndexerStackInfoSchema, +} from "../../../stack-info/zod-schemas/ensindexer-stack-info"; +import { IndexingMetadataContextStatusCodes } from "../indexing-metadata-context"; + +const makeSerializedIndexingMetadataContextUninitializedSchema = (valueLabel?: string) => { + const label = valueLabel ?? "SerializedIndexingMetadataContextUninitialized"; + + return z.object({ + statusCode: z.literal(IndexingMetadataContextStatusCodes.Uninitialized, { + error: `${label} must have status code ${IndexingMetadataContextStatusCodes.Uninitialized}`, + }), + }); +}; + +export const makeSerializedIndexingMetadataContextInitializedSchema = (valueLabel?: string) => { + const label = valueLabel ?? "SerializedIndexingMetadataContextInitialized"; + + return z.object({ + statusCode: z.literal(IndexingMetadataContextStatusCodes.Initialized, { + error: `${label} must have status code ${IndexingMetadataContextStatusCodes.Initialized}`, + }), + indexingStatus: makeSerializedCrossChainIndexingStatusSnapshotSchema(`${label}.indexingStatus`), + stackInfo: makeSerializedEnsIndexerStackInfoSchema(`${label}.stackInfo`), + }); +}; + +export const makeSerializedIndexingMetadataContextSchema = (valueLabel?: string) => { + const label = valueLabel ?? "SerializedIndexingMetadataContext"; + + return z.discriminatedUnion("statusCode", [ + makeSerializedIndexingMetadataContextUninitializedSchema(label), + makeSerializedIndexingMetadataContextInitializedSchema(label), + ]); +}; + +const makeIndexingMetadataContextUninitializedSchema = + makeSerializedIndexingMetadataContextUninitializedSchema; + +export const makeIndexingMetadataContextInitializedSchema = (valueLabel?: string) => { + const label = valueLabel ?? "IndexingMetadataContextInitialized"; + + return z.object({ + statusCode: z.literal(IndexingMetadataContextStatusCodes.Initialized, { + error: `${label} must have status code ${IndexingMetadataContextStatusCodes.Initialized}`, + }), + indexingStatus: makeCrossChainIndexingStatusSnapshotSchema(`${label}.indexingStatus`), + stackInfo: makeEnsIndexerStackInfoSchema(`${label}.stackInfo`), + }); +}; + +export const makeIndexingMetadataContextSchema = (valueLabel?: string) => { + const label = valueLabel ?? "IndexingMetadataContext"; + + return z.discriminatedUnion("statusCode", [ + makeIndexingMetadataContextUninitializedSchema(label), + makeIndexingMetadataContextInitializedSchema(label), + ]); +}; diff --git a/packages/integration-test-env/src/orchestrator.ts b/packages/integration-test-env/src/orchestrator.ts index 17e2a72d58..e713f44d91 100644 --- a/packages/integration-test-env/src/orchestrator.ts +++ b/packages/integration-test-env/src/orchestrator.ts @@ -39,7 +39,10 @@ import { } from "testcontainers"; import { ENSNamespaceIds } from "@ensnode/datasources"; -import { OmnichainIndexingStatusIds } from "@ensnode/ensnode-sdk"; +import { + IndexingMetadataContextStatusCodes, + OmnichainIndexingStatusIds, +} from "@ensnode/ensnode-sdk"; const MONOREPO_ROOT = resolve(import.meta.dirname, "../../.."); const DOCKER_DIR = resolve(MONOREPO_ROOT, "docker"); @@ -199,9 +202,14 @@ async function pollIndexingStatus( while (Date.now() - start < timeoutMs) { checkAborted(); try { - const snapshot = await ensDbClient.getIndexingStatusSnapshot(); - if (snapshot !== undefined) { - const omnichainStatus = snapshot.omnichainSnapshot.omnichainStatus; + const indexingMetadataContext = await ensDbClient.getIndexingMetadataContext(); + + if ( + indexingMetadataContext.statusCode === IndexingMetadataContextStatusCodes.Uninitialized + ) { + console.log("IndexingMetadataContext is uninitialized, waiting..."); + } else { + const { omnichainStatus } = indexingMetadataContext.indexingStatus.omnichainSnapshot; log(`Omnichain status: ${omnichainStatus}`); if ( omnichainStatus === OmnichainIndexingStatusIds.Following ||