Make e2e tests fail on network/auth errors

Author: brandonkachen

.agents/__tests__/context-pruner.integration.test.ts

@@ -11,17 +11,6 @@ import {
   type JSONValue,
 } from '@codebuff/sdk'
 
-const shouldSkip = (output: { type: string; message?: string; errorCode?: string }) => {
-  if (output.type !== 'error') return false
-  const msg = output.message?.toLowerCase() ?? ''
-  return (
-    output.errorCode === 'NETWORK_ERROR' ||
-    msg.includes('network error') ||
-    msg.includes('authentication') ||
-    msg.includes('api key')
-  )
-}
-
 /**
  * Integration tests for the context-pruner agent.
  * These tests verify that context-pruner correctly prunes message history
@@ -157,7 +146,6 @@ Do not do anything else. Just spawn context-pruner and then report the result.`,
       if (run.output.type === 'error') {
         console.error('Test 1 Error:', JSON.stringify(run.output, null, 2))
       }
-      if (shouldSkip(run.output)) return
       expect(run.output.type).not.toEqual('error')
 
       // Get the final message history from session state
@@ -286,7 +274,6 @@ Do not do anything else. Just spawn context-pruner and then report the result.`,
       if (run.output.type === 'error') {
         console.error('Test 2 Error:', JSON.stringify(run.output, null, 2))
       }
-      if (shouldSkip(run.output)) return
       expect(run.output.type).not.toEqual('error')
 
       // Get final messages and verify tool pairs are intact

.agents/__tests__/file-explorer.integration.test.ts

@@ -7,17 +7,6 @@ import fileListerDefinition from '../file-explorer/file-lister'
 
 import type { PrintModeEvent } from '@codebuff/common/types/print-mode'
 
-const shouldSkip = (output: { type: string; message?: string; errorCode?: string }) => {
-  if (output.type !== 'error') return false
-  const msg = output.message?.toLowerCase() ?? ''
-  return (
-    output.errorCode === 'NETWORK_ERROR' ||
-    msg.includes('network error') ||
-    msg.includes('authentication') ||
-    msg.includes('api key')
-  )
-}
-
 /**
  * Integration tests for agents that use the read_subtree tool.
  * These tests verify that the SDK properly initializes the session state
@@ -124,7 +113,6 @@ export interface User {
       })
 
       // The output should not be an error
-      if (shouldSkip(run.output)) return
       expect(run.output.type).not.toEqual('error')
 
       // Verify we got some output
@@ -189,7 +177,6 @@ export interface User {
         },
       })
 
-      if (shouldSkip(run.output)) return
       expect(run.output.type).not.toEqual('error')
 
       const outputStr =
@@ -244,7 +231,6 @@ export interface User {
         handleEvent: () => {},
       })
 
-      if (shouldSkip(run.output)) return
       expect(run.output.type).not.toEqual('error')
 
       const outputStr =

common/src/mcp/client.ts

@@ -6,7 +6,11 @@ import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/
 import type { MCPConfig } from '../types/mcp'
 import type { ToolResultOutput } from '../types/messages/content-part'
 import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js'
-import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'
+import type {
+  BlobResourceContents,
+  CallToolResult,
+  TextResourceContents,
+} from '@modelcontextprotocol/sdk/types.js'
 
 const runningClients: Record<string, Client> = {}
 const listToolsCache: Record<
@@ -104,6 +108,14 @@ export function listMCPTools(
   return listToolsCache[clientId]
 }
 
+function getResourceData(
+  resource: TextResourceContents | BlobResourceContents,
+): string {
+  if ('text' in resource) return resource.text
+  if ('blob' in resource) return resource.blob
+  return ''
+}
+
 export async function callMCPTool(
   clientId: string,
   ...args: Parameters<typeof Client.prototype.callTool>
@@ -136,15 +148,9 @@ export async function callMCPTool(
       } satisfies ToolResultOutput
     }
     if (c.type === 'resource') {
-      const resourceData =
-        'text' in c.resource
-          ? c.resource.text
-          : 'blob' in c.resource
-            ? c.resource.blob
-            : undefined
       return {
         type: 'media',
-        data: resourceData ?? '',
+        data: getResourceData(c.resource),
         mediaType: c.resource.mimeType ?? 'text/plain',
       } satisfies ToolResultOutput
     }

sdk/e2e/custom-agents/api-integration-agent.e2e.test.ts

@@ -12,7 +12,6 @@ import {
   EventCollector,
   getApiKey,
   skipIfNoApiKey,
-  shouldSkipOutput,
   DEFAULT_TIMEOUT,
 } from '../utils'
 
@@ -112,8 +111,6 @@ Summarize the response data clearly.`,
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
 
       const toolCalls = collector.getEventsByType('tool_call')
@@ -139,8 +136,6 @@ Summarize the response data clearly.`,
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       // Should complete without crashing
       expect(collector.hasEventType('finish')).toBe(true)
     },

sdk/e2e/custom-agents/database-query-agent.e2e.test.ts

@@ -12,7 +12,6 @@ import {
   EventCollector,
   getApiKey,
   skipIfNoApiKey,
-  shouldSkipOutput,
   MOCK_DATABASE,
   DEFAULT_TIMEOUT,
 } from '../utils'
@@ -83,8 +82,6 @@ Always format query results in a readable way.`,
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
 
       const toolCalls = collector.getEventsByType('tool_call')
@@ -115,8 +112,6 @@ Always format query results in a readable way.`,
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
       expect(collector.hasEventType('finish')).toBe(true)
     },

sdk/e2e/custom-agents/weather-agent.e2e.test.ts

@@ -12,7 +12,6 @@ import {
   EventCollector,
   getApiKey,
   skipIfNoApiKey,
-  shouldSkipOutput,
   MOCK_WEATHER_DATA,
   DEFAULT_TIMEOUT,
 } from '../utils'
@@ -75,8 +74,6 @@ Always report the temperature and conditions clearly.`,
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
 
       // Check that the tool was called
@@ -112,8 +109,6 @@ Always report the temperature and conditions clearly.`,
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
       expect(collector.hasEventType('finish')).toBe(true)
     },

sdk/e2e/features/knowledge-files.e2e.test.ts

@@ -11,7 +11,6 @@ import {
   EventCollector,
   getApiKey,
   skipIfNoApiKey,
-  shouldSkipOutput,
   DEFAULT_AGENT,
   DEFAULT_TIMEOUT,
 } from '../utils'
@@ -40,8 +39,6 @@ describe('Features: Knowledge Files', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
 
       const responseText = collector.getFullText().toUpperCase()
@@ -72,8 +69,6 @@ describe('Features: Knowledge Files', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
 
       const responseText = collector.getFullText().toLowerCase()

sdk/e2e/features/max-agent-steps.e2e.test.ts

@@ -11,7 +11,6 @@ import {
   EventCollector,
   getApiKey,
   skipIfNoApiKey,
-  shouldSkipOutput,
   DEFAULT_AGENT,
   DEFAULT_TIMEOUT,
 } from '../utils'
@@ -38,8 +37,6 @@ describe('Features: Max Agent Steps', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
       expect(collector.hasEventType('finish')).toBe(true)
     },
@@ -60,8 +57,6 @@ describe('Features: Max Agent Steps', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       // Should still complete for simple prompts
       expect(collector.hasEventType('start')).toBe(true)
     },

sdk/e2e/features/project-files.e2e.test.ts

@@ -11,7 +11,6 @@ import {
   EventCollector,
   getApiKey,
   skipIfNoApiKey,
-  shouldSkipOutput,
   SAMPLE_PROJECT_FILES,
   DEFAULT_AGENT,
   DEFAULT_TIMEOUT,
@@ -39,8 +38,6 @@ describe('Features: Project Files', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
 
       const responseText = collector.getFullText().toLowerCase()
@@ -69,8 +66,6 @@ describe('Features: Project Files', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       expect(result.output.type).not.toBe('error')
 
       const responseText = collector.getFullText().toLowerCase()

sdk/e2e/integration/event-ordering.integration.test.ts

@@ -12,7 +12,6 @@ import {
   EventCollector,
   getApiKey,
   skipIfNoApiKey,
-  shouldSkipOutput,
   DEFAULT_AGENT,
   DEFAULT_TIMEOUT,
 } from '../utils'
@@ -38,8 +37,6 @@ describe('Integration: Event Ordering', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       const startIndex = collector.events.findIndex((e) => e.type === 'start')
       expect(startIndex).toBe(0)
     },
@@ -59,8 +56,6 @@ describe('Integration: Event Ordering', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       const finishIndex = collector.events.findIndex((e) => e.type === 'finish')
       const lastTextIndex = collector.events
         .map((e, i) => (e.type === 'text' ? i : -1))
@@ -89,8 +84,6 @@ describe('Integration: Event Ordering', () => {
         cwd: process.cwd(),
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       const toolCalls = collector.getEventsByType('tool_call')
       const toolResults = collector.getEventsByType('tool_result')
 
@@ -121,8 +114,6 @@ describe('Integration: Event Ordering', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       // Use collector's verifyEventOrder method
       const hasCorrectOrder = collector.verifyEventOrder(['start', 'finish'])
       expect(hasCorrectOrder).toBe(true)
@@ -143,8 +134,6 @@ describe('Integration: Event Ordering', () => {
         handleEvent: collector.handleEvent,
       })
 
-      if (shouldSkipOutput(result.output)) return
-
       // Find the last finish event
       const finishEvents = collector.getEventsByType('finish')
       if (finishEvents.length > 0) {
@@ -173,16 +162,12 @@ describe('Integration: Event Ordering', () => {
         handleEvent: collector1.handleEvent,
       })
 
-      if (shouldSkipOutput(result1.output)) return
-
       const result2 = await client.run({
         agent: DEFAULT_AGENT,
         prompt: 'Say "second"',
         handleEvent: collector2.handleEvent,
       })
 
-      if (shouldSkipOutput(result2.output)) return
-
       // Both should have correct ordering
       expect(collector1.verifyEventOrder(['start', 'finish'])).toBe(true)
       expect(collector2.verifyEventOrder(['start', 'finish'])).toBe(true)