diff --git a/eslint-suppressions.json b/eslint-suppressions.json index b95add1c978..48b23fc66b3 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -348,7 +348,7 @@ }, "packages/assets-controllers/src/TokenListController.test.ts": { "@typescript-eslint/explicit-function-return-type": { - "count": 2 + "count": 1 }, "id-denylist": { "count": 2 @@ -357,14 +357,6 @@ "count": 7 } }, - "packages/assets-controllers/src/TokenListController.ts": { - "@typescript-eslint/explicit-function-return-type": { - "count": 6 - }, - "no-restricted-syntax": { - "count": 7 - } - }, "packages/assets-controllers/src/TokenRatesController.test.ts": { "@typescript-eslint/explicit-function-return-type": { "count": 4 diff --git a/packages/assets-controllers/CHANGELOG.md b/packages/assets-controllers/CHANGELOG.md index b5a267e3220..0d64d6152e6 100644 --- a/packages/assets-controllers/CHANGELOG.md +++ b/packages/assets-controllers/CHANGELOG.md @@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **BREAKING:** `TokenListController` now persists `tokensChainsCache` via `StorageService` using per-chain files to reduce write amplification ([#7413](https://github.com/MetaMask/core/pull/7413)) + - Each chain's token cache (~100-500KB) is stored in a separate file, avoiding rewriting all chains (~5MB) on every update + - Includes migration logic to automatically split existing cache data into per-chain files on first launch after upgrade + - All chains are loaded in parallel at startup to maintain compatibility with TokenDetectionController + - `tokensChainsCache` state metadata now has `persist: false` to prevent duplicate persistence - **BREAKING:** `AccountTrackerController` now requires `KeyringController:getState` action and `KeyringController:lock` event in addition to existing allowed actions and events ([#7492](https://github.com/MetaMask/core/pull/7492)) - Added `#isLocked` property to track keyring lock state, initialized from `KeyringController:getState` - Added `isActive` getter that returns `true` when keyring is unlocked and user is onboarded diff --git a/packages/assets-controllers/package.json b/packages/assets-controllers/package.json index 84fd988c86c..edebf771eee 100644 --- a/packages/assets-controllers/package.json +++ b/packages/assets-controllers/package.json @@ -78,6 +78,7 @@ "@metamask/snaps-controllers": "^14.0.1", "@metamask/snaps-sdk": "^9.0.0", "@metamask/snaps-utils": "^11.0.0", + "@metamask/storage-service": "^0.0.1", "@metamask/transaction-controller": "^62.7.0", "@metamask/utils": "^11.8.1", "@types/bn.js": "^5.1.5", diff --git a/packages/assets-controllers/src/TokenListController.test.ts b/packages/assets-controllers/src/TokenListController.test.ts index b6a03aa7b52..3cf80032978 100644 --- a/packages/assets-controllers/src/TokenListController.test.ts +++ b/packages/assets-controllers/src/TokenListController.test.ts @@ -22,6 +22,7 @@ import type { TokenListMap, TokenListState, TokenListControllerMessenger, + DataCache, } from './TokenListController'; import { TokenListController } from './TokenListController'; import { advanceTime } from '../../../tests/helpers'; @@ -478,8 +479,59 @@ type RootMessenger = Messenger< AllTokenListControllerEvents >; +// Mock storage for StorageService +const mockStorage = new Map(); + const getMessenger = (): RootMessenger => { - return new Messenger({ namespace: MOCK_ANY_NAMESPACE }); + const messenger = new Messenger({ namespace: MOCK_ANY_NAMESPACE }); + + // Register StorageService mock handlers + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).registerActionHandler( + 'StorageService:getItem', + (controllerNamespace: string, key: string) => { + const storageKey = `${controllerNamespace}:${key}`; + const value = mockStorage.get(storageKey); + return value ? { result: value } : {}; + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).registerActionHandler( + 'StorageService:setItem', + (controllerNamespace: string, key: string, value: unknown) => { + const storageKey = `${controllerNamespace}:${key}`; + mockStorage.set(storageKey, value); + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).registerActionHandler( + 'StorageService:removeItem', + (controllerNamespace: string, key: string) => { + const storageKey = `${controllerNamespace}:${key}`; + mockStorage.delete(storageKey); + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).registerActionHandler( + 'StorageService:getAllKeys', + (controllerNamespace: string) => { + const keys: string[] = []; + const prefix = `${controllerNamespace}:`; + mockStorage.forEach((_value, key) => { + // Only include keys for this namespace + if (key.startsWith(prefix)) { + const keyWithoutNamespace = key.substring(prefix.length); + keys.push(keyWithoutNamespace); + } + }); + return keys; + }, + ); + + return messenger; }; const getRestrictedMessenger = ( @@ -496,13 +548,24 @@ const getRestrictedMessenger = ( }); messenger.delegate({ messenger: tokenListControllerMessenger, - actions: ['NetworkController:getNetworkClientById'], + actions: [ + 'NetworkController:getNetworkClientById', + 'StorageService:getItem', + 'StorageService:setItem', + 'StorageService:removeItem', + 'StorageService:getAllKeys', + ], events: ['NetworkController:stateChange'], }); return tokenListControllerMessenger; }; describe('TokenListController', () => { + beforeEach(() => { + // Clear mock storage between tests + mockStorage.clear(); + }); + afterEach(() => { jest.clearAllTimers(); sinon.restore(); @@ -1069,7 +1132,7 @@ describe('TokenListController', () => { state: existingState, }); expect(controller.state).toStrictEqual(existingState); - controller.clearingTokenListData(); + await controller.clearingTokenListData(); expect(controller.state.tokensChainsCache).toStrictEqual({}); @@ -1135,6 +1198,96 @@ describe('TokenListController', () => { }); }); + it('should handle errors when clearing data on network change', async () => { + // Create messenger where getAllKeys throws during network change + const messenger = new Messenger({ + namespace: MOCK_ANY_NAMESPACE, + }); + + // Register getAllKeys to throw error + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).registerActionHandler( + 'StorageService:getAllKeys', + () => { + throw new Error('Failed to get keys during network change'); + }, + ); + + // Register other handlers + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).registerActionHandler( + 'StorageService:getItem', + () => ({}), + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).registerActionHandler( + 'StorageService:setItem', + (_controllerNamespace: string, _key: string, _value: unknown) => { + // Do nothing - testing error path + }, + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).registerActionHandler( + 'StorageService:removeItem', + (_controllerNamespace: string, _key: string) => { + // Do nothing - testing error path + }, + ); + + const getNetworkClientById = buildMockGetNetworkClientById({ + [InfuraNetworkType.mainnet]: buildInfuraNetworkClientConfiguration( + InfuraNetworkType.mainnet, + ), + [InfuraNetworkType.sepolia]: buildInfuraNetworkClientConfiguration( + InfuraNetworkType.sepolia, + ), + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).registerActionHandler( + 'NetworkController:getNetworkClientById', + getNetworkClientById, + ); + + const restrictedMessenger = getRestrictedMessenger(messenger); + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + preventPollingOnNetworkRestart: true, + messenger: restrictedMessenger, + }); + + // Wait for initialization + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Mock console.error to verify error handling + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(); + + // Trigger network change (should try to clear data and catch error) + // Using type assertion since we're testing with a minimal messenger + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messenger as any).publish( + 'NetworkController:stateChange', + { + selectedNetworkClientId: InfuraNetworkType.sepolia, + networkConfigurationsByChainId: {}, + networksMetadata: {}, + }, + [], + ); + + // Wait for async error handling + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Verify error was logged from clearingTokenListData's internal error handling + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'TokenListController: Failed to clear cache from storage:', + expect.any(Error), + ); + + consoleErrorSpy.mockRestore(); + controller.destroy(); + }); + describe('startPolling', () => { let clock: sinon.SinonFakeTimers; const pollingIntervalTime = 1000; @@ -1331,7 +1484,6 @@ describe('TokenListController', () => { ).toMatchInlineSnapshot(` Object { "preventPollingOnNetworkRestart": false, - "tokensChainsCache": Object {}, } `); }); @@ -1355,6 +1507,734 @@ describe('TokenListController', () => { `); }); }); + + describe('StorageService migration', () => { + it('should migrate tokensChainsCache from state to StorageService on first launch', async () => { + const messenger = getMessenger(); + const restrictedMessenger = getRestrictedMessenger(messenger); + + // Simulate old persisted state with tokensChainsCache + const oldPersistedState = { + tokensChainsCache: { + [ChainId.mainnet]: { + data: sampleMainnetTokensChainsCache, + timestamp: Date.now(), + }, + }, + preventPollingOnNetworkRestart: false, + }; + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + state: oldPersistedState, + }); + + // Wait for async migration to complete + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Verify data was migrated to StorageService (per-chain file) + const chainStorageKey = `tokensChainsCache:${ChainId.mainnet}`; + const { result } = await messenger.call( + 'StorageService:getItem', + 'TokenListController', + chainStorageKey, + ); + + expect(result).toBeDefined(); + const resultCache = result as DataCache; + expect(resultCache.data).toBeDefined(); + expect(resultCache.timestamp).toBeDefined(); + expect(resultCache.data).toStrictEqual(sampleMainnetTokensChainsCache); + + controller.destroy(); + }); + + it('should not overwrite StorageService if it already has data', async () => { + const messenger = getMessenger(); + const restrictedMessenger = getRestrictedMessenger(messenger); + + // Pre-populate StorageService with existing data (per-chain file) + const existingChainData: DataCache = { + data: sampleMainnetTokensChainsCache, + timestamp: Date.now(), + }; + const chainStorageKey = `tokensChainsCache:${ChainId.mainnet}`; + await messenger.call( + 'StorageService:setItem', + 'TokenListController', + chainStorageKey, + existingChainData, + ); + + // Initialize with different state data + const stateWithDifferentData = { + tokensChainsCache: { + [ChainId.mainnet]: { + data: sampleMainnetTokensChainsCache, + timestamp: Date.now(), + }, + }, + preventPollingOnNetworkRestart: false, + }; + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + state: stateWithDifferentData, + }); + + // Wait for migration logic to run + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Verify StorageService still has original data (not overwritten) + const { result } = await messenger.call( + 'StorageService:getItem', + 'TokenListController', + chainStorageKey, + ); + + expect(result).toStrictEqual(existingChainData); + const resultCache = result as DataCache; + expect(resultCache.data).toStrictEqual(existingChainData.data); + + controller.destroy(); + }); + + it('should not migrate when state has empty tokensChainsCache', async () => { + const messenger = getMessenger(); + const restrictedMessenger = getRestrictedMessenger(messenger); + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + state: { tokensChainsCache: {}, preventPollingOnNetworkRestart: false }, + }); + + // Wait for migration logic to run + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Verify nothing was saved to StorageService (check no per-chain files) + const allKeys = await messenger.call( + 'StorageService:getAllKeys', + 'TokenListController', + ); + const cacheKeys = allKeys.filter((key) => + key.startsWith('tokensChainsCache:'), + ); + + expect(cacheKeys).toHaveLength(0); + + controller.destroy(); + }); + + it('should save and load tokensChainsCache from StorageService', async () => { + const messenger = getMessenger(); + const restrictedMessenger = getRestrictedMessenger(messenger); + + // Create controller and fetch tokens (which saves to storage) + const controller1 = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + }); + + nock(tokenService.TOKEN_END_POINT_API) + .get(getTokensPath(ChainId.mainnet)) + .reply(200, sampleMainnetTokenList); + + await controller1.fetchTokenList(ChainId.mainnet); + const savedCache = controller1.state.tokensChainsCache; + + controller1.destroy(); + + // Verify data is in StorageService (per-chain file) + const chainStorageKey = `tokensChainsCache:${ChainId.mainnet}`; + const { result } = await messenger.call( + 'StorageService:getItem', + 'TokenListController', + chainStorageKey, + ); + + expect(result).toBeDefined(); + expect(result).toStrictEqual(savedCache[ChainId.mainnet]); + }); + + it('should save tokensChainsCache to StorageService when fetching tokens', async () => { + const messenger = getMessenger(); + const restrictedMessenger = getRestrictedMessenger(messenger); + + nock(tokenService.TOKEN_END_POINT_API) + .get(getTokensPath(ChainId.mainnet)) + .reply(200, sampleMainnetTokenList); + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + }); + + await controller.fetchTokenList(ChainId.mainnet); + + // Verify data was saved to StorageService (per-chain file) + const chainStorageKey = `tokensChainsCache:${ChainId.mainnet}`; + const { result } = await messenger.call( + 'StorageService:getItem', + 'TokenListController', + chainStorageKey, + ); + + expect(result).toBeDefined(); + const resultCache = result as DataCache; + expect(resultCache.data).toBeDefined(); + expect(resultCache.timestamp).toBeDefined(); + + controller.destroy(); + }); + + it('should clear tokensChainsCache from StorageService when clearing data', async () => { + const messenger = getMessenger(); + const restrictedMessenger = getRestrictedMessenger(messenger); + + // Pre-populate StorageService (per-chain file) + const chainData: DataCache = { + data: sampleMainnetTokensChainsCache, + timestamp: Date.now(), + }; + const chainStorageKey = `tokensChainsCache:${ChainId.mainnet}`; + await messenger.call( + 'StorageService:setItem', + 'TokenListController', + chainStorageKey, + chainData, + ); + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + state: { + tokensChainsCache: { + [ChainId.mainnet]: chainData, + }, + preventPollingOnNetworkRestart: false, + }, + }); + + // Wait a bit for async initialization to complete + await new Promise((resolve) => setTimeout(resolve, 50)); + + await controller.clearingTokenListData(); + + // Verify data was removed from StorageService (per-chain file removed) + const allKeys = await messenger.call( + 'StorageService:getAllKeys', + 'TokenListController', + ); + const cacheKeys = allKeys.filter((key) => + key.startsWith('tokensChainsCache:'), + ); + + expect(cacheKeys).toHaveLength(0); + expect(controller.state.tokensChainsCache).toStrictEqual({}); + + controller.destroy(); + }); + + it('should handle errors when loading individual chain cache files', async () => { + // Pre-populate storage with two chains + const validChainData: DataCache = { + data: sampleMainnetTokensChainsCache, + timestamp: Date.now(), + }; + const binanceChainData: DataCache = { + data: sampleBinanceTokensChainsCache, + timestamp: Date.now(), + }; + + mockStorage.set( + `TokenListController:tokensChainsCache:${ChainId.mainnet}`, + validChainData, + ); + mockStorage.set( + `TokenListController:tokensChainsCache:${ChainId.goerli}`, + binanceChainData, + ); + + // Create messenger with getItem that returns error for goerli + const messengerWithErrors = new Messenger({ + namespace: MOCK_ANY_NAMESPACE, + }); + + // Register getItem handler that returns error for goerli + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:getItem', + (controllerNamespace: string, key: string) => { + if (key === `tokensChainsCache:${ChainId.goerli}`) { + return { error: 'Failed to load chain data' }; + } + const storageKey = `${controllerNamespace}:${key}`; + const value = mockStorage.get(storageKey); + return value ? { result: value } : {}; + }, + ); + + // Register other handlers normally + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:setItem', + (controllerNamespace: string, key: string, value: unknown) => { + const storageKey = `${controllerNamespace}:${key}`; + mockStorage.set(storageKey, value); + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:removeItem', + (controllerNamespace: string, key: string) => { + const storageKey = `${controllerNamespace}:${key}`; + mockStorage.delete(storageKey); + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:getAllKeys', + (controllerNamespace: string) => { + const keys: string[] = []; + const prefix = `${controllerNamespace}:`; + mockStorage.forEach((_value, key) => { + // Only include keys for this namespace + if (key.startsWith(prefix)) { + const keyWithoutNamespace = key.substring(prefix.length); + keys.push(keyWithoutNamespace); + } + }); + return keys; + }, + ); + + const restrictedMessenger = getRestrictedMessenger(messengerWithErrors); + + // Mock console.error to verify it's called for the error case + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(); + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + }); + + // Wait for async loading to complete + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Verify that mainnet chain loaded successfully + expect(controller.state.tokensChainsCache[ChainId.mainnet]).toBeDefined(); + expect( + controller.state.tokensChainsCache[ChainId.mainnet].data, + ).toStrictEqual(sampleMainnetTokensChainsCache); + + // Verify that goerli chain is not in the cache (due to error) + expect( + controller.state.tokensChainsCache[ChainId.goerli], + ).toBeUndefined(); + + // Verify console.error was called with the error + expect(consoleErrorSpy).toHaveBeenCalledWith( + `TokenListController: Error loading cache for ${ChainId.goerli}:`, + 'Failed to load chain data', + ); + + consoleErrorSpy.mockRestore(); + controller.destroy(); + }); + + it('should handle StorageService errors when saving cache', async () => { + // Create a messenger with setItem that throws errors + const messengerWithErrors = new Messenger({ + namespace: MOCK_ANY_NAMESPACE, + }); + + // Register all handlers, but make setItem throw + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:getItem', + (controllerNamespace: string, key: string) => { + const storageKey = `${controllerNamespace}:${key}`; + const value = mockStorage.get(storageKey); + return value ? { result: value } : {}; + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:setItem', + () => { + throw new Error('Storage write failed'); + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:removeItem', + (controllerNamespace: string, key: string) => { + const storageKey = `${controllerNamespace}:${key}`; + mockStorage.delete(storageKey); + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:getAllKeys', + (controllerNamespace: string) => { + const keys: string[] = []; + const prefix = `${controllerNamespace}:`; + mockStorage.forEach((_value, key) => { + // Only include keys for this namespace + if (key.startsWith(prefix)) { + const keyWithoutNamespace = key.substring(prefix.length); + keys.push(keyWithoutNamespace); + } + }); + return keys; + }, + ); + + const restrictedMessenger = getRestrictedMessenger(messengerWithErrors); + + // Mock console.error to verify it's called for save errors + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(); + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + }); + + // Wait for async initialization + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Try to fetch tokens - this should trigger save which will fail + nock(tokenService.TOKEN_END_POINT_API) + .get(getTokensPath(ChainId.mainnet)) + .reply(200, sampleMainnetTokenList); + + await controller.fetchTokenList(ChainId.mainnet); + + // Verify console.error was called with the save error + expect(consoleErrorSpy).toHaveBeenCalledWith( + `TokenListController: Failed to save cache for ${ChainId.mainnet}:`, + expect.any(Error), + ); + + // Verify state was still updated even though save failed + expect(controller.state.tokensChainsCache[ChainId.mainnet]).toBeDefined(); + + consoleErrorSpy.mockRestore(); + controller.destroy(); + }); + + it('should handle errors during migration to StorageService', async () => { + // Create messenger where getAllKeys throws to cause migration logic to fail + const messengerWithErrors = new Messenger({ + namespace: MOCK_ANY_NAMESPACE, + }); + + // Register getItem to return empty (no old storage data) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:getItem', + () => { + return {}; // No old single-file storage + }, + ); + + // Register setItem normally + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:setItem', + (controllerNamespace: string, key: string, value: unknown) => { + const storageKey = `${controllerNamespace}:${key}`; + mockStorage.set(storageKey, value); + }, + ); + + // Register getAllKeys to throw error during migration check + // This will cause the migration logic itself to fail (not just the save) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:getAllKeys', + () => { + throw new Error('Failed to get keys during migration'); + }, + ); + + // Register removeItem (not used in this test but required) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:removeItem', + () => { + // Do nothing + }, + ); + + const restrictedMessenger = getRestrictedMessenger(messengerWithErrors); + + // Mock console.error to verify it's called for migration errors + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(); + + // Initialize with state data that will trigger migration + const stateWithData = { + tokensChainsCache: { + [ChainId.mainnet]: { + data: sampleMainnetTokensChainsCache, + timestamp: Date.now(), + }, + }, + preventPollingOnNetworkRestart: false, + }; + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + state: stateWithData, + }); + + // Wait for async migration to attempt and fail + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Verify console.error was called with the migration error + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'TokenListController: Failed to migrate cache to storage:', + expect.any(Error), + ); + + consoleErrorSpy.mockRestore(); + controller.destroy(); + }); + + it('should handle errors when clearing cache from StorageService', async () => { + // Create messenger where getAllKeys throws only during clear + const messengerWithErrors = new Messenger({ + namespace: MOCK_ANY_NAMESPACE, + }); + + let shouldThrow = false; + // Register getAllKeys to throw error only when flag is set + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:getAllKeys', + () => { + if (shouldThrow) { + throw new Error('Failed to get keys'); + } + return []; // Return empty array for initialization + }, + ); + + // Register other handlers + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:getItem', + () => ({}), + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:setItem', + (_controllerNamespace: string, _key: string, _value: unknown) => { + // Do nothing - testing error path in getAllKeys + }, + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (messengerWithErrors as any).registerActionHandler( + 'StorageService:removeItem', + (_controllerNamespace: string, _key: string) => { + // Do nothing - testing error path in getAllKeys + }, + ); + + const restrictedMessenger = getRestrictedMessenger(messengerWithErrors); + + // Initialize controller with pre-populated state + const initialCache = { + [ChainId.mainnet]: { + timestamp: Date.now(), + data: sampleMainnetTokensChainsCache, + }, + }; + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + state: { tokensChainsCache: initialCache }, + }); + + // Wait for initialization + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Verify cache exists before clearing + expect( + Object.keys(controller.state.tokensChainsCache).length, + ).toBeGreaterThan(0); + + // Now enable throwing to test error handling during clear + shouldThrow = true; + + // Mock console.error + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(); + + // Try to clear - should catch error but still clear state + await controller.clearingTokenListData(); + + // Verify error was logged + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'TokenListController: Failed to clear cache from storage:', + expect.any(Error), + ); + + // Verify state was still cleared despite the error + // This ensures consistent behavior with the no-keys case + expect(controller.state.tokensChainsCache).toStrictEqual({}); + + consoleErrorSpy.mockRestore(); + controller.destroy(); + }); + + it('should only load cache from storage once even when fetchTokenList is called multiple times', async () => { + // Pre-populate storage with cached data + const chainData: DataCache = { + data: sampleMainnetTokensChainsCache, + timestamp: Date.now(), + }; + mockStorage.set( + `TokenListController:tokensChainsCache:${ChainId.mainnet}`, + chainData, + ); + + // Track how many times getItem is called + let getItemCallCount = 0; + let getAllKeysCallCount = 0; + + const trackingMessenger = new Messenger({ + namespace: MOCK_ANY_NAMESPACE, + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (trackingMessenger as any).registerActionHandler( + 'StorageService:getItem', + (controllerNamespace: string, key: string) => { + getItemCallCount += 1; + const storageKey = `${controllerNamespace}:${key}`; + const value = mockStorage.get(storageKey); + return value ? { result: value } : {}; + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (trackingMessenger as any).registerActionHandler( + 'StorageService:setItem', + (controllerNamespace: string, key: string, value: unknown) => { + const storageKey = `${controllerNamespace}:${key}`; + mockStorage.set(storageKey, value); + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (trackingMessenger as any).registerActionHandler( + 'StorageService:removeItem', + (controllerNamespace: string, key: string) => { + const storageKey = `${controllerNamespace}:${key}`; + mockStorage.delete(storageKey); + }, + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (trackingMessenger as any).registerActionHandler( + 'StorageService:getAllKeys', + (controllerNamespace: string) => { + getAllKeysCallCount += 1; + const keys: string[] = []; + const prefix = `${controllerNamespace}:`; + mockStorage.forEach((_value, key) => { + if (key.startsWith(prefix)) { + const keyWithoutNamespace = key.substring(prefix.length); + keys.push(keyWithoutNamespace); + } + }); + return keys; + }, + ); + + const restrictedMessenger = getRestrictedMessenger(trackingMessenger); + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + }); + + // Wait for initialization to complete + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Record call counts after initialization + const getItemCallsAfterInit = getItemCallCount; + const getAllKeysCallsAfterInit = getAllKeysCallCount; + + // getAllKeys should be called twice during init (once for load, once for migration check) + expect(getAllKeysCallsAfterInit).toBe(2); + // getItem should be called once for the cached chain during load + expect(getItemCallsAfterInit).toBe(1); + + // Now call fetchTokenList multiple times + nock(tokenService.TOKEN_END_POINT_API) + .get(getTokensPath(ChainId.mainnet)) + .reply(200, sampleMainnetTokenList) + .persist(); + + await controller.fetchTokenList(ChainId.mainnet); + await controller.fetchTokenList(ChainId.mainnet); + await controller.fetchTokenList(ChainId.mainnet); + + // Verify getAllKeys was NOT called again after initialization + // (getItem may be called for other reasons, but getAllKeys is only used in load/migrate) + expect(getAllKeysCallCount).toBe(getAllKeysCallsAfterInit); + + controller.destroy(); + }); + }); + + describe('deprecated methods', () => { + it('should restart polling when restart() is called', async () => { + const messenger = getMessenger(); + const restrictedMessenger = getRestrictedMessenger(messenger); + + const controller = new TokenListController({ + chainId: ChainId.mainnet, + messenger: restrictedMessenger, + interval: 100, + }); + + nock(tokenService.TOKEN_END_POINT_API) + .get(getTokensPath(ChainId.mainnet)) + .reply(200, sampleMainnetTokenList) + .persist(); + + // Start initial polling + await controller.start(); + + // Wait for first fetch + await new Promise((resolve) => setTimeout(resolve, 150)); + + const initialCache = { ...controller.state.tokensChainsCache }; + expect(initialCache[ChainId.mainnet]).toBeDefined(); + + // Restart polling + await controller.restart(); + + // Wait for another fetch + await new Promise((resolve) => setTimeout(resolve, 150)); + + // Verify polling continued + expect(controller.state.tokensChainsCache[ChainId.mainnet]).toBeDefined(); + + controller.destroy(); + }); + }); }); /** @@ -1363,7 +2243,7 @@ describe('TokenListController', () => { * @param chainId - The chain ID. * @returns The constructed path. */ -function getTokensPath(chainId: Hex) { +function getTokensPath(chainId: Hex): string { return `/tokens/${convertHexToDecimal( chainId, )}?occurrenceFloor=3&includeNativeAssets=false&includeTokenFees=false&includeAssetType=false&includeERC20Permit=false&includeStorage=false`; diff --git a/packages/assets-controllers/src/TokenListController.ts b/packages/assets-controllers/src/TokenListController.ts index 6ee57c476bb..6b18a1f0449 100644 --- a/packages/assets-controllers/src/TokenListController.ts +++ b/packages/assets-controllers/src/TokenListController.ts @@ -11,6 +11,12 @@ import type { NetworkControllerGetNetworkClientByIdAction, } from '@metamask/network-controller'; import { StaticIntervalPollingController } from '@metamask/polling-controller'; +import type { + StorageServiceSetItemAction, + StorageServiceGetItemAction, + StorageServiceRemoveItemAction, + StorageServiceGetAllKeysAction, +} from '@metamask/storage-service'; import type { Hex } from '@metamask/utils'; import { Mutex } from 'async-mutex'; @@ -38,7 +44,7 @@ export type TokenListToken = { export type TokenListMap = Record; -type DataCache = { +export type DataCache = { timestamp: number; data: TokenListMap; }; @@ -65,7 +71,12 @@ export type GetTokenListState = ControllerGetStateAction< export type TokenListControllerActions = GetTokenListState; -type AllowedActions = NetworkControllerGetNetworkClientByIdAction; +type AllowedActions = + | NetworkControllerGetNetworkClientByIdAction + | StorageServiceSetItemAction + | StorageServiceGetItemAction + | StorageServiceRemoveItemAction + | StorageServiceGetAllKeysAction; type AllowedEvents = NetworkControllerStateChangeEvent; @@ -78,7 +89,7 @@ export type TokenListControllerMessenger = Messenger< const metadata: StateMetadata = { tokensChainsCache: { includeInStateLogs: false, - persist: true, + persist: false, // Persisted separately via StorageService includeInDebugSnapshot: true, usedInUi: true, }, @@ -110,17 +121,36 @@ export class TokenListController extends StaticIntervalPollingController { - private readonly mutex = new Mutex(); + readonly #mutex = new Mutex(); + + /** + * Promise that resolves when initialization (loading cache from storage) is complete. + * Methods that access the cache should await this before proceeding. + */ + readonly #initializationPromise: Promise; + + // Storage key prefix for per-chain files + static readonly #storageKeyPrefix = 'tokensChainsCache'; + + /** + * Get storage key for a specific chain. + * + * @param chainId - The chain ID. + * @returns Storage key for the chain. + */ + static #getChainStorageKey(chainId: Hex): string { + return `${TokenListController.#storageKeyPrefix}:${chainId}`; + } - private intervalId?: ReturnType; + #intervalId?: ReturnType; - private readonly intervalDelay: number; + readonly #intervalDelay: number; - private readonly cacheRefreshThreshold: number; + readonly #cacheRefreshThreshold: number; - private chainId: Hex; + #chainId: Hex; - private abortController: AbortController; + #abortController: AbortController; /** * Creates a TokenListController instance. @@ -159,12 +189,18 @@ export class TokenListController extends StaticIntervalPollingController { + const releaseLock = await this.#mutex.acquire(); + try { + await this.#loadCacheFromStorage(); + await this.#migrateStateToStorage(); + } catch (error) { + console.error( + 'TokenListController: Failed to initialize from storage:', + error, + ); + } finally { + releaseLock(); + } + } + + /** + * Load tokensChainsCache from StorageService into state. + * Loads all cached chains from separate per-chain files in parallel. + * Called during initialization to restore cached data. + * + * Note: This method merges loaded data with existing state to avoid + * overwriting any fresh data that may have been fetched concurrently. + * Caller must hold the mutex. + * + * @returns A promise that resolves when loading is complete. + */ + async #loadCacheFromStorage(): Promise { + try { + // Get all keys for this controller + const allKeys = await this.messenger.call( + 'StorageService:getAllKeys', + name, + ); + + // Filter keys that belong to tokensChainsCache (per-chain files) + const cacheKeys = allKeys.filter((key) => + key.startsWith(`${TokenListController.#storageKeyPrefix}:`), + ); + + if (cacheKeys.length === 0) { + return; // No cached data + } + + // Load all chains in parallel + const chainCaches = await Promise.all( + cacheKeys.map(async (key) => { + // Extract chainId from key: 'tokensChainsCache:0x1' → '0x1' + const chainId = key.split(':')[1] as Hex; + + const { result, error } = await this.messenger.call( + 'StorageService:getItem', + name, + key, + ); + + if (error) { + console.error( + `TokenListController: Error loading cache for ${chainId}:`, + error, + ); + return null; + } + + return result ? { chainId, data: result as DataCache } : null; + }), + ); + + // Build complete cache from loaded chains + const loadedCache: TokensChainsCache = {}; + chainCaches.forEach((chainCache) => { + if (chainCache) { + loadedCache[chainCache.chainId] = chainCache.data; + } + }); + + // Merge loaded cache with existing state, preferring existing data + // (which may be fresher if fetched during initialization) + if (Object.keys(loadedCache).length > 0) { + this.update((state) => { + // Only load chains that don't already exist in state + // This prevents overwriting fresh API data with stale cached data + for (const [chainId, cacheData] of Object.entries(loadedCache)) { + if (!state.tokensChainsCache[chainId as Hex]) { + state.tokensChainsCache[chainId as Hex] = cacheData; + } + } + }); + } + } catch (error) { + console.error( + 'TokenListController: Failed to load cache from storage:', + error, + ); + } + } + + /** + * Save a specific chain's cache to StorageService. + * This persists only the updated chain's data, reducing write amplification. + * + * @param chainId - The chain ID to save. + * @returns A promise that resolves when saving is complete. + */ + async #saveChainCacheToStorage(chainId: Hex): Promise { + try { + const chainData = this.state.tokensChainsCache[chainId]; + + if (!chainData) { + console.warn(`TokenListController: No cache data for chain ${chainId}`); + return; + } + + const storageKey = TokenListController.#getChainStorageKey(chainId); + + await this.messenger.call( + 'StorageService:setItem', + name, + storageKey, + chainData, + ); + } catch (error) { + console.error( + `TokenListController: Failed to save cache for ${chainId}:`, + error, + ); + } + } + + /** + * Migrate tokensChainsCache from old persisted state to per-chain files. + * Handles backward compatibility for users upgrading from the old + * framework-managed state (persist: true) to StorageService. + * + * Only migrates chains that exist in state but not in storage. If any + * chain fails to save, it will be logged and the cache will self-heal + * when fetchTokenList is called for that chain. + * + * @returns A promise that resolves when migration is complete. + */ + async #migrateStateToStorage(): Promise { + try { + // Check if we have data in state that needs migration + const chainsInState = Object.keys(this.state.tokensChainsCache) as Hex[]; + if (chainsInState.length === 0) { + return; // No data to migrate + } + + // Get existing per-chain files to determine which chains still need migration + const allKeys = await this.messenger.call( + 'StorageService:getAllKeys', + name, + ); + const existingChainKeys = new Set( + allKeys.filter((key) => + key.startsWith(`${TokenListController.#storageKeyPrefix}:`), + ), + ); + + // Find chains that exist in state but not in storage (need migration) + const chainsMissingFromStorage = chainsInState.filter((chainId) => { + const storageKey = TokenListController.#getChainStorageKey(chainId); + return !existingChainKeys.has(storageKey); + }); + + if (chainsMissingFromStorage.length === 0) { + return; // All chains already migrated + } + + // Migrate only the chains that are missing from storage + console.log( + `TokenListController: Migrating ${chainsMissingFromStorage.length} chain(s) from persisted state to per-chain storage`, + ); + + // Migrate chains in parallel. Individual failures are logged inside + // #saveChainCacheToStorage. If any fail, the cache will self-heal + // when fetchTokenList is called for that chain. + await Promise.all( + chainsMissingFromStorage.map((chainId) => + this.#saveChainCacheToStorage(chainId), + ), + ); + + console.log( + 'TokenListController: Migration to per-chain storage complete', + ); + } catch (error) { + console.error( + 'TokenListController: Failed to migrate cache to storage:', + error, + ); + } + } + /** * Updates state and restarts polling on changes to the network controller * state. * * @param networkControllerState - The updated network controller state. */ - async #onNetworkControllerStateChange(networkControllerState: NetworkState) { + async #onNetworkControllerStateChange( + networkControllerState: NetworkState, + ): Promise { const selectedNetworkClient = this.messenger.call( 'NetworkController:getNetworkClientById', networkControllerState.selectedNetworkClientId, ); const { chainId } = selectedNetworkClient.configuration; - if (this.chainId !== chainId) { - this.abortController.abort(); - this.abortController = new AbortController(); - this.chainId = chainId; + if (this.#chainId !== chainId) { + this.#abortController.abort(); + this.#abortController = new AbortController(); + this.#chainId = chainId; if (this.state.preventPollingOnNetworkRestart) { - this.clearingTokenListData(); + this.clearingTokenListData().catch((error) => { + console.error('Failed to clear token list data:', error); + }); } } } @@ -214,8 +453,8 @@ export class TokenListController extends StaticIntervalPollingController { + if (!isTokenListSupportedForNetwork(this.#chainId)) { return; } await this.#startDeprecatedPolling(); @@ -227,8 +466,8 @@ export class TokenListController extends StaticIntervalPollingController { + this.#stopPolling(); await this.#startDeprecatedPolling(); } @@ -238,8 +477,8 @@ export class TokenListController extends StaticIntervalPollingController { // renaming this to avoid collision with base class - await safelyExecute(() => this.fetchTokenList(this.chainId)); + await safelyExecute(() => this.fetchTokenList(this.#chainId)); // TODO: Either fix this lint violation or explain why it's necessary to ignore. // eslint-disable-next-line @typescript-eslint/no-misused-promises - this.intervalId = setInterval(async () => { - await safelyExecute(() => this.fetchTokenList(this.chainId)); - }, this.intervalDelay); + this.#intervalId = setInterval(async () => { + await safelyExecute(() => this.fetchTokenList(this.#chainId)); + }, this.#intervalDelay); } /** @@ -293,12 +532,17 @@ export class TokenListController extends StaticIntervalPollingController { - const releaseLock = await this.mutex.acquire(); + // Wait for initialization to complete before fetching + // This ensures we have loaded any cached data from storage first + await this.#initializationPromise; + + const releaseLock = await this.#mutex.acquire(); try { if (this.isCacheValid(chainId)) { return; @@ -309,7 +553,7 @@ export class TokenListController extends StaticIntervalPollingController fetchTokenListByChainId( chainId, - this.abortController.signal, + this.#abortController.signal, ) as Promise, ); @@ -328,22 +572,35 @@ export class TokenListController extends StaticIntervalPollingController { - const newDataCache: DataCache = { data: {}, timestamp: Date.now() }; - state.tokensChainsCache[chainId] ??= newDataCache; - state.tokensChainsCache[chainId].data = tokenList; - state.tokensChainsCache[chainId].timestamp = Date.now(); + state.tokensChainsCache[chainId] = newDataCache; }); + + // Persist only this chain to StorageService (reduces write amplification) + await this.#saveChainCacheToStorage(chainId); return; } - // No response - fallback to previous state, or initialise empty + // No response - fallback to previous state, or initialise empty. + // Only initialize with a new timestamp if there's no existing cache. + // If there's existing cache, keep it as-is without updating the timestamp + // to avoid making stale data appear "fresh" and preventing retry attempts. if (!tokensFromAPI) { - this.update((state) => { + const existingCache = this.state.tokensChainsCache[chainId]; + if (!existingCache) { + // No existing cache - initialize empty and persist const newDataCache: DataCache = { data: {}, timestamp: Date.now() }; - state.tokensChainsCache[chainId] ??= newDataCache; - state.tokensChainsCache[chainId].timestamp = Date.now(); - }); + this.update((state) => { + state.tokensChainsCache[chainId] = newDataCache; + }); + await this.#saveChainCacheToStorage(chainId); + } + // If there's existing cache, keep it as-is (don't update timestamp or persist) } } finally { releaseLock(); @@ -355,20 +612,96 @@ export class TokenListController extends StaticIntervalPollingController { - return { - ...this.state, - tokensChainsCache: {}, - }; - }); + async clearingTokenListData(): Promise { + // Wait for initialization to complete before clearing + await this.#initializationPromise; + + const releaseLock = await this.#mutex.acquire(); + try { + const allKeys = await this.messenger.call( + 'StorageService:getAllKeys', + name, + ); + + // Filter and remove all tokensChainsCache keys + const cacheKeys = allKeys.filter((key) => + key.startsWith(`${TokenListController.#storageKeyPrefix}:`), + ); + + if (cacheKeys.length === 0) { + // No storage keys to remove, just clear state + this.update((state) => { + state.tokensChainsCache = {}; + }); + return; + } + + // Use Promise.allSettled to handle partial failures gracefully. + // This ensures all removals are attempted and we can track which succeeded. + const results = await Promise.allSettled( + cacheKeys.map((key) => + this.messenger.call('StorageService:removeItem', name, key), + ), + ); + + // Identify which chains failed to be removed from storage + const failedChainIds = new Set(); + results.forEach((result, index) => { + if (result.status === 'rejected') { + const key = cacheKeys[index]; + const chainId = key.split(':')[1] as Hex; + failedChainIds.add(chainId); + console.error( + `TokenListController: Failed to remove cache for chain ${chainId}:`, + result.reason, + ); + } + }); + + // Update state to match storage: keep only chains that failed to be removed + this.update((state) => { + if (failedChainIds.size === 0) { + state.tokensChainsCache = {}; + } else { + // Keep only chains that failed to be removed from storage + const preservedCache: TokensChainsCache = {}; + for (const chainId of failedChainIds) { + if (state.tokensChainsCache[chainId]) { + preservedCache[chainId] = state.tokensChainsCache[chainId]; + } + } + state.tokensChainsCache = preservedCache; + } + }); + } catch (error) { + console.error( + 'TokenListController: Failed to clear cache from storage:', + error, + ); + // Still clear state even if storage access fails. + // This maintains consistency with the no-keys case (lines 646-651) + // and fulfills the JSDoc contract that state will be cleared. + this.update((state) => { + state.tokensChainsCache = {}; + }); + } finally { + releaseLock(); + } } /** diff --git a/packages/assets-controllers/tsconfig.build.json b/packages/assets-controllers/tsconfig.build.json index 5ef0e52c3b8..d24f83bf4e8 100644 --- a/packages/assets-controllers/tsconfig.build.json +++ b/packages/assets-controllers/tsconfig.build.json @@ -18,6 +18,7 @@ { "path": "../preferences-controller/tsconfig.build.json" }, { "path": "../polling-controller/tsconfig.build.json" }, { "path": "../permission-controller/tsconfig.build.json" }, + { "path": "../storage-service/tsconfig.build.json" }, { "path": "../transaction-controller/tsconfig.build.json" }, { "path": "../phishing-controller/tsconfig.build.json" } ], diff --git a/packages/assets-controllers/tsconfig.json b/packages/assets-controllers/tsconfig.json index a537b98ca39..5a4630d0a47 100644 --- a/packages/assets-controllers/tsconfig.json +++ b/packages/assets-controllers/tsconfig.json @@ -18,6 +18,7 @@ { "path": "../phishing-controller" }, { "path": "../polling-controller" }, { "path": "../permission-controller" }, + { "path": "../storage-service" }, { "path": "../transaction-controller" } ], "include": ["../../types", "./src", "../../tests"] diff --git a/yarn.lock b/yarn.lock index 8a184300a9d..9101faaadef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2662,6 +2662,7 @@ __metadata: "@metamask/snaps-controllers": "npm:^14.0.1" "@metamask/snaps-sdk": "npm:^9.0.0" "@metamask/snaps-utils": "npm:^11.0.0" + "@metamask/storage-service": "npm:^0.0.1" "@metamask/transaction-controller": "npm:^62.7.0" "@metamask/utils": "npm:^11.8.1" "@ts-bridge/cli": "npm:^0.6.4" @@ -4941,7 +4942,7 @@ __metadata: languageName: node linkType: hard -"@metamask/storage-service@workspace:packages/storage-service": +"@metamask/storage-service@npm:^0.0.1, @metamask/storage-service@workspace:packages/storage-service": version: 0.0.0-use.local resolution: "@metamask/storage-service@workspace:packages/storage-service" dependencies: