From 69d49e389459aec7e7a641e6cda40d0c981b4f2a Mon Sep 17 00:00:00 2001 From: Anastasia Rodionova Date: Mon, 9 Jun 2025 21:30:15 +0200 Subject: [PATCH 1/3] Implement getMany for meta and abi sql store --- .../transaction-decoder/src/sql/abi-store.ts | 87 +++++++++++++++++++ .../src/sql/contract-meta-store.ts | 59 +++++++++++++ 2 files changed, 146 insertions(+) diff --git a/packages/transaction-decoder/src/sql/abi-store.ts b/packages/transaction-decoder/src/sql/abi-store.ts index 9676c52..1c2ca96 100644 --- a/packages/transaction-decoder/src/sql/abi-store.ts +++ b/packages/transaction-decoder/src/sql/abi-store.ts @@ -139,6 +139,93 @@ export const make = (strategies: AbiStore.AbiStore['strategies']) => result: null, } }), + + getMany: (keys) => + Effect.gen(function* () { + if (keys.length === 0) { + return [] + } + + // Build a batch query for all the keys + const conditions = keys.map(({ address, signature, event, chainID }) => { + const addressQuery = sql.and([ + sql`address = ${address.toLowerCase()}`, + sql`chain = ${chainID}`, + sql`type = 'address'`, + ]) + + const signatureQuery = signature + ? sql.and([sql`signature = ${signature}`, sql`type = 'func'`]) + : undefined + const eventQuery = event ? sql.and([sql`event = ${event}`, sql`type = 'event'`]) : undefined + + return signature == null && event == null + ? addressQuery + : sql.or([addressQuery, signatureQuery, eventQuery].filter(Boolean)) + }) + + const batchQuery = sql.or(conditions) + + const allItems = yield* sql`SELECT * FROM ${table} WHERE ${batchQuery}`.pipe( + Effect.tapError(Effect.logError), + Effect.catchAll(() => Effect.succeed([])), + ) + + // Process results for each key + return keys.map(({ address, signature, event, chainID }) => { + const keyItems = allItems.filter((item) => { + // Match by address and chain + if ( + typeof item.address === 'string' && + item.address.toLowerCase() === address.toLowerCase() && + item.chain === chainID + ) { + return true + } + // Match by signature + if (signature && item.signature === signature && item.type === 'func') { + return true + } + // Match by event + if (event && item.event === event && item.type === 'event') { + return true + } + return false + }) + + const successItems = keyItems.filter((item) => item.status === 'success') + + const item = + successItems.find((item) => { + // Prioritize address over fragments + return item.type === 'address' + }) ?? successItems[0] + + if (item != null) { + return { + status: 'success', + result: { + type: item.type, + event: item.event, + signature: item.signature, + address, + chainID, + abi: item.abi, + }, + } as AbiStore.ContractAbiResult + } else if (keyItems[0] != null && keyItems[0].status === 'not-found') { + return { + status: 'not-found', + result: null, + } + } + + return { + status: 'empty', + result: null, + } + }) + }), }) }), ) diff --git a/packages/transaction-decoder/src/sql/contract-meta-store.ts b/packages/transaction-decoder/src/sql/contract-meta-store.ts index 824d048..0e59f7a 100644 --- a/packages/transaction-decoder/src/sql/contract-meta-store.ts +++ b/packages/transaction-decoder/src/sql/contract-meta-store.ts @@ -107,6 +107,65 @@ export const make = (strategies: ContractMetaStore.ContractMetaStore['strategies result: null, } }), + getMany: (params) => + Effect.gen(function* () { + if (params.length === 0) { + return [] + } + + // Build WHERE conditions for batch select using OR clauses + const whereConditions = params.map(({ address, chainID }) => + sql.and([sql`address = ${address.toLowerCase()}`, sql`chain = ${chainID}`]), + ) + + const items = yield* sql` + SELECT * FROM ${table} + WHERE ${sql.or(whereConditions)} + `.pipe( + Effect.tapError(Effect.logError), + Effect.catchAll(() => Effect.succeed([])), + ) + + // Create a map for O(1) lookup of results + const itemsMap = new Map() + items.forEach((item) => { + const address = + typeof item.address === 'string' ? item.address.toLowerCase() : String(item.address).toLowerCase() + const key = `${address}-${item.chain}` + itemsMap.set(key, item) + }) + + // Build results array in the same order as input params + return params.map(({ address, chainID }) => { + const key = `${address.toLowerCase()}-${chainID}` + const item = itemsMap.get(key) + + if (item && item.status === 'success') { + return { + status: 'success', + result: { + contractAddress: address, + contractName: item.contract_name, + tokenSymbol: item.token_symbol, + decimals: item.decimals, + type: item.type, + address, + chainID, + } as ContractData, + } + } else if (item && item.status === 'not-found') { + return { + status: 'not-found', + result: null, + } + } + + return { + status: 'empty', + result: null, + } + }) as ContractMetaStore.ContractMetaResult[] + }), }) }), ) From 60b2309bb68ffebf59104edd0e3d8d74eb87189a Mon Sep 17 00:00:00 2001 From: Anastasia Rodionova Date: Tue, 10 Jun 2025 10:43:54 +0200 Subject: [PATCH 2/3] Add few optimizations --- .../transaction-decoder/src/sql/abi-store.ts | 226 +++++++++--------- .../src/sql/contract-meta-store.ts | 150 ++++++------ 2 files changed, 188 insertions(+), 188 deletions(-) diff --git a/packages/transaction-decoder/src/sql/abi-store.ts b/packages/transaction-decoder/src/sql/abi-store.ts index 1c2ca96..ffd1927 100644 --- a/packages/transaction-decoder/src/sql/abi-store.ts +++ b/packages/transaction-decoder/src/sql/abi-store.ts @@ -2,6 +2,92 @@ import * as AbiStore from '../abi-store.js' import { Effect, Layer } from 'effect' import { SqlClient } from '@effect/sql' +// Utility function to build query conditions for a single key +const buildQueryForKey = ( + sql: SqlClient.SqlClient, + { address, signature, event, chainID }: { address: string; signature?: string; event?: string; chainID: number }, +) => { + const addressQuery = sql.and([ + sql`address = ${address.toLowerCase()}`, + sql`chain = ${chainID}`, + sql`type = 'address'`, + ]) + + const signatureQuery = signature ? sql.and([sql`signature = ${signature}`, sql`type = 'func'`]) : undefined + const eventQuery = event ? sql.and([sql`event = ${event}`, sql`type = 'event'`]) : undefined + + return signature == null && event == null + ? addressQuery + : sql.or([addressQuery, signatureQuery, eventQuery].filter(Boolean)) +} + +// Convert database items to result format +const createResult = (items: readonly any[], address: string, chainID: number): AbiStore.ContractAbiResult => { + const successItems = items.filter((item) => item.status === 'success') + + const item = + successItems.find((item) => { + // Prioritize address over fragments + return item.type === 'address' + }) ?? successItems[0] + + if (item != null) { + return { + status: 'success', + result: { + type: item.type, + event: item.event, + signature: item.signature, + address, + chainID, + abi: item.abi, + }, + } as AbiStore.ContractAbiResult + } else if (items[0] != null && items[0].status === 'not-found') { + return { + status: 'not-found', + result: null, + } + } + + return { + status: 'empty', + result: null, + } +} + +// Build single lookup map with prefixed keys +const buildLookupMap = (allItems: readonly any[]) => { + const lookupMap = new Map() + + const addToMap = (key: string, item: any) => { + if (!lookupMap.has(key)) lookupMap.set(key, []) + lookupMap.get(key)?.push(item) + } + + for (const item of allItems) { + // Address-based lookup: "addr:address_chain" (with same type check as original) + if (typeof item.address === 'string' && typeof item.chain === 'number') { + const addressKey = `addr:${item.address.toLowerCase()}_${item.chain}` + addToMap(addressKey, item) + } + + // Signature-based lookup: "sig:signature" + if (item.signature && item.type === 'func') { + const signatureKey = `sig:${item.signature}` + addToMap(signatureKey, item) + } + + // Event-based lookup: "event:event" + if (item.event && item.type === 'event') { + const eventKey = `event:${item.event}` + addToMap(eventKey, item) + } + } + + return lookupMap +} + export const make = (strategies: AbiStore.AbiStore['strategies']) => Layer.scoped( AbiStore.AbiStore, @@ -89,81 +175,22 @@ export const make = (strategies: AbiStore.AbiStore['strategies']) => get: ({ address, signature, event, chainID }) => Effect.gen(function* () { - const addressQuery = sql.and([ - sql`address = ${address.toLowerCase()}`, - sql`chain = ${chainID}`, - sql`type = 'address'`, - ]) - - const signatureQuery = signature ? sql.and([sql`signature = ${signature}`, sql`type = 'func'`]) : undefined - const eventQuery = event ? sql.and([sql`event = ${event}`, sql`type = 'event'`]) : undefined - const query = - signature == null && event == null - ? addressQuery - : sql.or([addressQuery, signatureQuery, eventQuery].filter(Boolean)) + const query = buildQueryForKey(sql, { address, signature, event, chainID }) const items = yield* sql` SELECT * FROM ${table} WHERE ${query}`.pipe( Effect.tapError(Effect.logError), Effect.catchAll(() => Effect.succeed([])), ) - const successItems = items.filter((item) => item.status === 'success') - - const item = - successItems.find((item) => { - // Prioritize address over fragments - return item.type === 'address' - }) ?? successItems[0] - - if (item != null) { - return { - status: 'success', - result: { - type: item.type, - event: item.event, - signature: item.signature, - address, - chainID, - abi: item.abi, - }, - } as AbiStore.ContractAbiResult - } else if (items[0] != null && items[0].status === 'not-found') { - return { - status: 'not-found', - result: null, - } - } - - return { - status: 'empty', - result: null, - } + return createResult(items, address, chainID) }), getMany: (keys) => Effect.gen(function* () { - if (keys.length === 0) { - return [] - } - - // Build a batch query for all the keys - const conditions = keys.map(({ address, signature, event, chainID }) => { - const addressQuery = sql.and([ - sql`address = ${address.toLowerCase()}`, - sql`chain = ${chainID}`, - sql`type = 'address'`, - ]) - - const signatureQuery = signature - ? sql.and([sql`signature = ${signature}`, sql`type = 'func'`]) - : undefined - const eventQuery = event ? sql.and([sql`event = ${event}`, sql`type = 'event'`]) : undefined - - return signature == null && event == null - ? addressQuery - : sql.or([addressQuery, signatureQuery, eventQuery].filter(Boolean)) - }) + if (keys.length === 0) return [] + // Single database query for all keys + const conditions = keys.map((key) => buildQueryForKey(sql, key)) const batchQuery = sql.or(conditions) const allItems = yield* sql`SELECT * FROM ${table} WHERE ${batchQuery}`.pipe( @@ -171,59 +198,32 @@ export const make = (strategies: AbiStore.AbiStore['strategies']) => Effect.catchAll(() => Effect.succeed([])), ) - // Process results for each key + // Build efficient lookup map once + const lookupMap = buildLookupMap(allItems) + + // Process results for each key using lookup map return keys.map(({ address, signature, event, chainID }) => { - const keyItems = allItems.filter((item) => { - // Match by address and chain - if ( - typeof item.address === 'string' && - item.address.toLowerCase() === address.toLowerCase() && - item.chain === chainID - ) { - return true - } - // Match by signature - if (signature && item.signature === signature && item.type === 'func') { - return true - } - // Match by event - if (event && item.event === event && item.type === 'event') { - return true - } - return false - }) - - const successItems = keyItems.filter((item) => item.status === 'success') - - const item = - successItems.find((item) => { - // Prioritize address over fragments - return item.type === 'address' - }) ?? successItems[0] - - if (item != null) { - return { - status: 'success', - result: { - type: item.type, - event: item.event, - signature: item.signature, - address, - chainID, - abi: item.abi, - }, - } as AbiStore.ContractAbiResult - } else if (keyItems[0] != null && keyItems[0].status === 'not-found') { - return { - status: 'not-found', - result: null, - } + const keyItems: any[] = [] + + // Get address-based matches + const addressKey = `addr:${address.toLowerCase()}_${chainID}` + const addressItems = lookupMap.get(addressKey) || [] + keyItems.push(...addressItems) + + // Get signature-based matches + if (signature) { + const signatureKey = `sig:${signature}` + const signatureItems = lookupMap.get(signatureKey) || [] + keyItems.push(...signatureItems) } - return { - status: 'empty', - result: null, + // Get event-based matches + if (event) { + const eventKey = `event:${event}` + const eventItems = lookupMap.get(eventKey) || [] + keyItems.push(...eventItems) } + return createResult(keyItems, address, chainID) }) }), }) diff --git a/packages/transaction-decoder/src/sql/contract-meta-store.ts b/packages/transaction-decoder/src/sql/contract-meta-store.ts index 0e59f7a..504625e 100644 --- a/packages/transaction-decoder/src/sql/contract-meta-store.ts +++ b/packages/transaction-decoder/src/sql/contract-meta-store.ts @@ -3,6 +3,60 @@ import { Effect, Layer } from 'effect' import * as ContractMetaStore from '../contract-meta-store.js' import { ContractData } from '../types.js' +// Utility function to build query conditions for a single key +const buildQueryForKey = (sql: SqlClient.SqlClient, { address, chainID }: { address: string; chainID: number }) => { + return sql.and([sql`address = ${address.toLowerCase()}`, sql`chain = ${chainID}`]) +} + +// Convert database items to result format +const createResult = ( + items: readonly any[], + address: string, + chainID: number, +): ContractMetaStore.ContractMetaResult => { + const successItems = items.filter((item) => item.status === 'success') + const item = successItems[0] + + if (item != null && item.status === 'success') { + return { + status: 'success', + result: { + contractAddress: address, + contractName: item.contract_name, + tokenSymbol: item.token_symbol, + decimals: item.decimals, + type: item.type, + address, + chainID, + } as ContractData, + } + } else if (items[0] != null && items[0].status === 'not-found') { + return { + status: 'not-found', + result: null, + } + } + + return { + status: 'empty', + result: null, + } +} + +// Build lookup map for efficient batch processing +const buildLookupMap = (allItems: readonly any[]) => { + const lookupMap = new Map() + + for (const item of allItems) { + if (typeof item.address === 'string' && typeof item.chain === 'number') { + const key = `${item.address.toLowerCase()}-${item.chain}` + lookupMap.set(key, item) + } + } + + return lookupMap +} + export const make = (strategies: ContractMetaStore.ContractMetaStore['strategies']) => Layer.effect( ContractMetaStore.ContractMetaStore, @@ -68,103 +122,49 @@ export const make = (strategies: ContractMetaStore.ContractMetaStore['strategies Effect.tapError(Effect.logError), Effect.catchAll(() => Effect.succeed(null)), ), + get: ({ address, chainID }) => Effect.gen(function* () { + const query = buildQueryForKey(sql, { address, chainID }) + const items = yield* sql` - SELECT * FROM ${table} - WHERE ${sql.and([sql`address = ${address.toLowerCase()}`, sql`chain = ${chainID}`])} - `.pipe( + SELECT * FROM ${table} + WHERE ${query} + `.pipe( Effect.tapError(Effect.logError), Effect.catchAll(() => Effect.succeed([])), ) - const successItems = items.filter((item) => item.status === 'success') - - const item = successItems[0] - - if (item != null && item.status === 'success') { - return { - status: 'success', - result: { - contractAddress: address, - contractName: item.contract_name, - tokenSymbol: item.token_symbol, - decimals: item.decimals, - type: item.type, - address, - chainID, - } as ContractData, - } - } else if (items[0] != null && items[0].status === 'not-found') { - return { - status: 'not-found', - result: null, - } - } - - return { - status: 'empty', - result: null, - } + return createResult(items, address, chainID) }), + getMany: (params) => Effect.gen(function* () { - if (params.length === 0) { - return [] - } + if (params.length === 0) return [] - // Build WHERE conditions for batch select using OR clauses - const whereConditions = params.map(({ address, chainID }) => - sql.and([sql`address = ${address.toLowerCase()}`, sql`chain = ${chainID}`]), - ) + // Single database query for all keys + const conditions = params.map((key) => buildQueryForKey(sql, key)) + const batchQuery = sql.or(conditions) - const items = yield* sql` + const allItems = yield* sql` SELECT * FROM ${table} - WHERE ${sql.or(whereConditions)} + WHERE ${batchQuery} `.pipe( Effect.tapError(Effect.logError), Effect.catchAll(() => Effect.succeed([])), ) - // Create a map for O(1) lookup of results - const itemsMap = new Map() - items.forEach((item) => { - const address = - typeof item.address === 'string' ? item.address.toLowerCase() : String(item.address).toLowerCase() - const key = `${address}-${item.chain}` - itemsMap.set(key, item) - }) + // Build efficient lookup map once + const lookupMap = buildLookupMap(allItems) - // Build results array in the same order as input params + // Process results for each key using lookup map return params.map(({ address, chainID }) => { const key = `${address.toLowerCase()}-${chainID}` - const item = itemsMap.get(key) + const item = lookupMap.get(key) + const items = item ? [item] : [] - if (item && item.status === 'success') { - return { - status: 'success', - result: { - contractAddress: address, - contractName: item.contract_name, - tokenSymbol: item.token_symbol, - decimals: item.decimals, - type: item.type, - address, - chainID, - } as ContractData, - } - } else if (item && item.status === 'not-found') { - return { - status: 'not-found', - result: null, - } - } - - return { - status: 'empty', - result: null, - } - }) as ContractMetaStore.ContractMetaResult[] + return createResult(items, address, chainID) + }) }), }) }), From bb9ee40585cea3139ed18447e4b6af1af1946242 Mon Sep 17 00:00:00 2001 From: Anastasia Rodionova Date: Thu, 17 Jul 2025 11:28:22 +0200 Subject: [PATCH 3/3] Changeset --- .changeset/light-impalas-agree.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/light-impalas-agree.md diff --git a/.changeset/light-impalas-agree.md b/.changeset/light-impalas-agree.md new file mode 100644 index 0000000..8abe824 --- /dev/null +++ b/.changeset/light-impalas-agree.md @@ -0,0 +1,5 @@ +--- +'@3loop/transaction-decoder': patch +--- + +Implement getMany for meta and abi sql store