diff --git a/scripts/erase-cassettes.sh b/scripts/erase-cassettes.sh new file mode 100755 index 0000000..8a0ddd6 --- /dev/null +++ b/scripts/erase-cassettes.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +cd "$(dirname "$(readlink -f "${BASH_SOURCE}")")"/.. + +rm -rf src/test/resources/cassettes/ +mkdir -p src/test/resources/cassettes/ diff --git a/scripts/record-cassettes.sh b/scripts/record-cassettes.sh new file mode 100755 index 0000000..9267728 --- /dev/null +++ b/scripts/record-cassettes.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +cd "$(dirname "$(readlink -f "${BASH_SOURCE}")")"/.. + +export VCR_MODE=record +./gradlew test diff --git a/src/test/java/dev/braintrust/TestHarness.java b/src/test/java/dev/braintrust/TestHarness.java index aa139b7..87a3235 100644 --- a/src/test/java/dev/braintrust/TestHarness.java +++ b/src/test/java/dev/braintrust/TestHarness.java @@ -1,7 +1,6 @@ package dev.braintrust; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import dev.braintrust.api.BraintrustApiClient; import dev.braintrust.config.BraintrustConfig; @@ -14,7 +13,6 @@ import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.logs.SdkLoggerProvider; import io.opentelemetry.sdk.metrics.SdkMeterProvider; -import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; @@ -27,6 +25,18 @@ import lombok.experimental.Accessors; public class TestHarness { + private static final VCR vcr; + + static { + vcr = + new VCR( + java.util.Map.of( + "https://api.openai.com/v1", "openai", + "https://api.anthropic.com", "anthropic", + "https://generativelanguage.googleapis.com", "google")); + vcr.start(); + Runtime.getRuntime().addShutdownHook(new Thread(vcr::stop)); + } public static TestHarness setup() { return setup(createTestConfig()); @@ -75,13 +85,12 @@ public static synchronized TestHarness setup(BraintrustConfig config) { @Accessors(fluent = true) private final Braintrust braintrust; - private final @Nonnull InMemorySpanExporter spanExporter; + private final @Nonnull UnitTestSpanExporter spanExporter; private TestHarness(@Nonnull Braintrust braintrust) { this.braintrust = braintrust; - var tracerBuilder = SdkTracerProvider.builder(); - this.spanExporter = InMemorySpanExporter.create(); + this.spanExporter = new UnitTestSpanExporter(); var loggerBuilder = SdkLoggerProvider.builder(); var meterBuilder = SdkMeterProvider.builder(); braintrust.openTelemetryEnable(tracerBuilder, loggerBuilder, meterBuilder); @@ -102,6 +111,30 @@ private TestHarness(@Nonnull Braintrust braintrust) { this.openTelemetry = openTelemetry; } + public String openAiBaseUrl() { + return vcr.getUrlForTargetBase("https://api.openai.com/v1"); + } + + public String openAiApiKey() { + return getEnv("OPENAI_API_KEY", "test-key"); + } + + public String anthropicBaseUrl() { + return vcr.getUrlForTargetBase("https://api.anthropic.com"); + } + + public String anthropicApiKey() { + return getEnv("ANTHROPIC_API_KEY", "test-key"); + } + + public String googleBaseUrl() { + return vcr.getUrlForTargetBase("https://generativelanguage.googleapis.com"); + } + + public String googleApiKey() { + return getEnv("GOOGLE_API_KEY", getEnv("GEMINI_API_KEY", "test-key")); + } + /** flush all pending spans and return all spans which have been exported so far */ public List awaitExportedSpans() { assertTrue( @@ -120,21 +153,7 @@ public List awaitExportedSpans() { */ @SneakyThrows public List awaitExportedSpans(int minSpanCount) { - var spans = awaitExportedSpans(); - int attempts = 0; - while (spans.size() < minSpanCount) { - attempts++; - if (attempts > 30) { - fail( - String.format( - "Timeout waiting for spans: expected at least %d spans, but got %d" - + " after %d attempts", - minSpanCount, spans.size(), attempts)); - } - Thread.sleep(1000); - spans = awaitExportedSpans(); - } - return spans; + return spanExporter.getFinishedSpanItems(minSpanCount); } private static BraintrustApiClient.InMemoryImpl createApiClient() { @@ -163,4 +182,9 @@ public static BraintrustConfig createTestConfig() { "BRAINTRUST_APP_URL", "https://testhost:3000", "BRAINTRUST_DEFAULT_PROJECT_NAME", defaultProjectName()); } + + private static String getEnv(String envarName, String defaultValue) { + var envar = System.getenv(envarName); + return envar == null ? defaultValue : envar; + } } diff --git a/src/test/java/dev/braintrust/UnitTestSpanExporter.java b/src/test/java/dev/braintrust/UnitTestSpanExporter.java new file mode 100644 index 0000000..19818fc --- /dev/null +++ b/src/test/java/dev/braintrust/UnitTestSpanExporter.java @@ -0,0 +1,84 @@ +package dev.braintrust; + +import static org.junit.jupiter.api.Assertions.fail; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import lombok.SneakyThrows; + +/** + * an in memory span exporter which allows for blocking until the exported span count reaches an + * expected min + */ +public class UnitTestSpanExporter implements SpanExporter { + private final Queue finishedSpanItems = new ConcurrentLinkedQueue<>(); + private final Lock lock = new ReentrantLock(); + private final Condition spansAdded = lock.newCondition(); + private boolean isStopped = false; + + public UnitTestSpanExporter() {} + + @SneakyThrows + public List getFinishedSpanItems(int minSpanCount) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + lock.lock(); + try { + while (finishedSpanItems.size() < minSpanCount) { + long remainingNanos = deadline - System.nanoTime(); + if (remainingNanos <= 0) { + fail( + String.format( + "Timeout waiting for spans: expected at least %d spans, but got" + + " %d after 30 seconds", + minSpanCount, finishedSpanItems.size())); + } + spansAdded.awaitNanos(remainingNanos); + } + return Collections.unmodifiableList(new ArrayList<>(finishedSpanItems)); + } finally { + lock.unlock(); + } + } + + public List getFinishedSpanItems() { + return Collections.unmodifiableList(new ArrayList<>(finishedSpanItems)); + } + + public void reset() { + finishedSpanItems.clear(); + } + + @Override + public CompletableResultCode export(Collection spans) { + if (isStopped) { + return CompletableResultCode.ofFailure(); + } + lock.lock(); + try { + finishedSpanItems.addAll(spans); + spansAdded.signalAll(); + } finally { + lock.unlock(); + } + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + finishedSpanItems.clear(); + isStopped = true; + return CompletableResultCode.ofSuccess(); + } +} diff --git a/src/test/java/dev/braintrust/VCR.java b/src/test/java/dev/braintrust/VCR.java new file mode 100644 index 0000000..61b8f09 --- /dev/null +++ b/src/test/java/dev/braintrust/VCR.java @@ -0,0 +1,271 @@ +package dev.braintrust; + +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.recording.RecordSpecBuilder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import opennlp.tools.commons.ThreadSafe; + +/** VCR (Video Cassette Recorder) for recording and replaying HTTP interactions. */ +@Slf4j +@ThreadSafe // implementation locks under the instance lock +public class VCR { + public enum VcrMode { + /** Playback from recorded cassettes (default) */ + REPLAY, + /** Record interactions to cassettes */ + RECORD, + /** Disable VCR (proxy calls but do not record) */ + OFF + } + + private static final String CASSETTES_ROOT = "src/test/resources/cassettes/"; + + private final Map proxyMap; + private final VcrMode mode; + private final Map targetUrlToMappingsDir; + private boolean recordingStarted = false; + + public VCR(Map targetUrlToCassettesDir) { + this( + VcrMode.valueOf(System.getenv().getOrDefault("VCR_MODE", "replay").toUpperCase()), + targetUrlToCassettesDir); + } + + private VCR(VcrMode mode, Map targetUrlToCassettesDir) { + this.mode = mode; + this.targetUrlToMappingsDir = Map.copyOf(targetUrlToCassettesDir); + + // Create a WireMockServer for each provider + this.proxyMap = new LinkedHashMap<>(); + for (Map.Entry entry : targetUrlToCassettesDir.entrySet()) { + String targetUrl = entry.getKey(); + String mappingsDir = entry.getValue(); + String cassettesDir = CASSETTES_ROOT + mappingsDir; + + createDirectoryStructure(cassettesDir); + + WireMockServer wireMock = + new WireMockServer( + wireMockConfig().dynamicPort().usingFilesUnderDirectory(cassettesDir)); + proxyMap.put(targetUrl, wireMock); + } + } + + private void createDirectoryStructure(String baseDir) { + try { + Path mappingsDir = Paths.get(baseDir, "mappings"); + Path filesDir = Paths.get(baseDir, "__files"); + Files.createDirectories(mappingsDir); + Files.createDirectories(filesDir); + } catch (Exception e) { + log.warn("Failed to create directory structure: {}", e.getMessage()); + } + } + + public synchronized void start() { + for (WireMockServer wireMock : proxyMap.values()) { + if (!wireMock.isRunning()) { + wireMock.start(); + } + } + startRecording(); + } + + public synchronized void stop() { + stopRecording(); + for (WireMockServer wireMock : proxyMap.values()) { + if (wireMock.isRunning()) { + wireMock.stop(); + } + } + } + + /** + * Get the URL for the targetUrl + * + *

In REPLAY/RECORD modes: returns WireMock URL (e.g. https://api.openai.com/v1 --> + * http://localhost:1234) + * + *

In OFF mode: returns the real target URL to bypass WireMock entirely + */ + public synchronized String getUrlForTargetBase(String targetUrl) { + assertStarted(); + if (mode == VcrMode.OFF) { + return targetUrl; + } + WireMockServer wireMock = proxyMap.get(targetUrl); + if (wireMock == null) { + throw new IllegalArgumentException("Unknown target URL: " + targetUrl); + } + return wireMock.baseUrl(); + } + + private void startRecording() { + if (mode == VcrMode.RECORD && !recordingStarted) { + targetUrlToMappingsDir.keySet().forEach(this::startRecording); + recordingStarted = true; + } else if (mode == VcrMode.REPLAY) { + log.info("️VCR_MODE=replay: Using recorded stubs from cassettes/"); + loadAndCreateProgrammaticStubs(); + } else if (mode == VcrMode.OFF) { + targetUrlToMappingsDir + .keySet() + .forEach(url -> log.info("VCR_MODE=off: wiring clients directly to {}", url)); + } + } + + private void startRecording(String targetBaseUrl) { + WireMockServer wireMock = proxyMap.get(targetBaseUrl); + if (wireMock == null) { + throw new IllegalArgumentException("Unknown target URL: " + targetBaseUrl); + } + + String mappingsDir = targetUrlToMappingsDir.get(targetBaseUrl); + log.info("VCR_MODE=record: Proxying to {}", targetBaseUrl); + log.info("Cassettes will be saved to: cassettes/{}", mappingsDir); + + RecordSpecBuilder recordSpec = + new RecordSpecBuilder() + .forTarget(targetBaseUrl) + .captureHeader("Content-Type") + // .captureHeader("Authorization", true) + .extractTextBodiesOver(0) // Always extract bodies + .makeStubsPersistent(true); // Save to disk + + wireMock.startRecording(recordSpec); + } + + /** + * Load recorded JSON mappings and create programmatic stubs for SSE responses. This avoids + * WireMock's issues with SSE playback from JSON mappings. + */ + private void loadAndCreateProgrammaticStubs() { + for (Map.Entry entry : targetUrlToMappingsDir.entrySet()) { + String targetUrl = entry.getKey(); + String mappingsDir = entry.getValue(); + WireMockServer wireMock = proxyMap.get(targetUrl); + + try { + Path mappingsDirPath = Paths.get(CASSETTES_ROOT, mappingsDir, "mappings"); + if (!Files.exists(mappingsDirPath)) { + continue; + } + + Files.walk(mappingsDirPath) + .filter(path -> path.toString().endsWith(".json")) + .forEach( + path -> { + try { + createProgrammaticStubFromMapping( + path, mappingsDir, wireMock); + } catch (Exception e) { + System.err.println( + "Failed to load mapping " + + path + + ": " + + e.getMessage()); + } + }); + } catch (Exception e) { + System.err.println( + "Failed to load programmatic stubs for " + + targetUrl + + ": " + + e.getMessage()); + } + } + } + + private void createProgrammaticStubFromMapping( + Path mappingPath, String mappingsDir, WireMockServer wireMock) throws Exception { + String json = Files.readString(mappingPath); + com.fasterxml.jackson.databind.ObjectMapper mapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + com.fasterxml.jackson.databind.JsonNode mapping = mapper.readTree(json); + + // Check if this is an SSE response + com.fasterxml.jackson.databind.JsonNode contentType = + mapping.at("/response/headers/Content-Type"); + boolean isSse = + contentType.isTextual() && contentType.asText().contains("text/event-stream"); + + if (!isSse) { + return; // Let WireMock handle non-SSE responses normally + } + + log.info("Creating programmatic stub for SSE response: " + mappingPath.getFileName()); + + // Extract request matching criteria + String url = mapping.at("/request/url").asText(); + String method = mapping.at("/request/method").asText(); + + // Extract request body pattern for matching + com.fasterxml.jackson.databind.JsonNode bodyPatterns = mapping.at("/request/bodyPatterns"); + + // Extract response body + String body; + if (mapping.at("/response/body").isTextual()) { + body = mapping.at("/response/body").asText(); + } else if (mapping.at("/response/bodyFileName").isTextual()) { + String bodyFileName = mapping.at("/response/bodyFileName").asText(); + Path bodyPath = Paths.get(CASSETTES_ROOT, mappingsDir, "__files", bodyFileName); + body = Files.readString(bodyPath); + } else { + return; + } + + // Create programmatic stub (like BraintrustOpenAITest does) + com.github.tomakehurst.wiremock.client.MappingBuilder stub = + com.github.tomakehurst.wiremock.client.WireMock.request( + method, com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo(url)); + + // Add body pattern matching if present + if (bodyPatterns.isArray() && !bodyPatterns.isEmpty()) { + com.fasterxml.jackson.databind.JsonNode firstPattern = bodyPatterns.get(0); + if (firstPattern.has("equalToJson")) { + String expectedJson = firstPattern.get("equalToJson").asText(); + stub.withRequestBody( + com.github.tomakehurst.wiremock.client.WireMock.equalToJson( + expectedJson, true, true)); + } + } + + com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder response = + com.github.tomakehurst.wiremock.client.WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "text/event-stream") + .withBody(body); + + // Use instance method + wireMock.stubFor(stub.willReturn(response)); + } + + private void stopRecording() { + if (mode == VcrMode.RECORD && recordingStarted) { + for (Map.Entry entry : proxyMap.entrySet()) { + String targetUrl = entry.getKey(); + WireMockServer wireMock = entry.getValue(); + wireMock.stopRecording(); + log.info("Recording saved for {}", targetUrl); + } + recordingStarted = false; + } + // Note: We don't stop the WireMock servers here because they're shared across tests. + // The servers will be stopped when the JVM shuts down via the shutdown hook. + } + + private void assertStarted() { + for (WireMockServer wireMock : proxyMap.values()) { + if (!wireMock.isRunning()) { + throw new IllegalStateException("VCR not started. See start() method"); + } + } + } +} diff --git a/src/test/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropicTest.java b/src/test/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropicTest.java index b4d2764..5f758cf 100644 --- a/src/test/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropicTest.java +++ b/src/test/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropicTest.java @@ -1,7 +1,5 @@ package dev.braintrust.instrumentation.anthropic; -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.junit.jupiter.api.Assertions.*; import com.anthropic.client.AnthropicClient; @@ -9,66 +7,30 @@ import com.anthropic.models.messages.MessageCreateParams; import com.anthropic.models.messages.Model; import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import dev.braintrust.TestHarness; import io.opentelemetry.api.common.AttributeKey; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; public class BraintrustAnthropicTest { private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - @RegisterExtension - static WireMockExtension wireMock = - WireMockExtension.newInstance().options(wireMockConfig().dynamicPort()).build(); - private TestHarness testHarness; @BeforeEach void beforeEach() { testHarness = TestHarness.setup(); - wireMock.resetAll(); } @Test @SneakyThrows void testWrapAnthropic() { - // Mock the Anthropic API response - wireMock.stubFor( - post(urlEqualTo("/v1/messages")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody( - """ - { - "id": "msg_test123", - "type": "message", - "role": "assistant", - "model": "claude-3-5-haiku-20241022", - "content": [ - { - "type": "text", - "text": "The capital of France is Paris." - } - ], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 20, - "output_tokens": 8 - } - } - """))); - - // Create Anthropic client pointing to WireMock server + // Create Anthropic client using VCR AnthropicClient anthropicClient = AnthropicOkHttpClient.builder() - .baseUrl("http://localhost:" + wireMock.getPort()) - .apiKey("test-api-key") + .baseUrl(testHarness.anthropicBaseUrl()) + .apiKey(testHarness.anthropicApiKey()) .build(); // Wrap with Braintrust instrumentation @@ -87,11 +49,10 @@ void testWrapAnthropic() { // Verify the response assertNotNull(response); - wireMock.verify(1, postRequestedFor(urlEqualTo("/v1/messages"))); - assertEquals("msg_test123", response.id()); + assertNotNull(response.id()); var contentBlock = response.content().get(0); assertTrue(contentBlock.isText()); - assertEquals("The capital of France is Paris.", contentBlock.asText().text()); + assertNotNull(contentBlock.asText().text()); // Verify spans were exported var spans = testHarness.awaitExportedSpans(); @@ -104,27 +65,19 @@ void testWrapAnthropic() { assertEquals( "claude-3-5-haiku-20241022", span.getAttributes().get(AttributeKey.stringKey("gen_ai.request.model"))); - assertEquals( - "claude-3-5-haiku-20241022", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model"))); - assertEquals( - "[end_turn]", + assertNotNull(span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model"))); + assertNotNull( span.getAttributes() - .get(AttributeKey.stringArrayKey("gen_ai.response.finish_reasons")) - .toString()); + .get(AttributeKey.stringArrayKey("gen_ai.response.finish_reasons"))); assertEquals( "anthropic.messages.create", span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name"))); - assertEquals( - "msg_test123", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); + assertNotNull(span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); assertEquals( "project_name:" + TestHarness.defaultProjectName(), span.getAttributes().get(AttributeKey.stringKey("braintrust.parent"))); - assertEquals( - 20L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); - assertEquals( - 8L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); + assertNotNull(span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); + assertNotNull(span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); assertEquals( 0.0, span.getAttributes().get(AttributeKey.doubleKey("gen_ai.request.temperature"))); @@ -132,11 +85,7 @@ void testWrapAnthropic() { 50L, span.getAttributes().get(AttributeKey.longKey("gen_ai.request.max_tokens"))); // Verify Braintrust-specific attributes - assertEquals( - "[{\"content\":\"What is the capital of" - + " France?\",\"role\":\"user\",\"valid\":true},{\"content\":\"You are a" - + " helpful assistant\",\"role\":\"system\",\"valid\":false}]", - span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json"))); + assertNotNull(span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json"))); // Verify output JSON String outputJson = @@ -144,14 +93,12 @@ void testWrapAnthropic() { assertNotNull(outputJson); var outputMessage = JSON_MAPPER.readTree(outputJson); - assertEquals("msg_test123", outputMessage.get("id").asText()); + assertNotNull(outputMessage.get("id")); assertEquals("message", outputMessage.get("type").asText()); assertEquals("assistant", outputMessage.get("role").asText()); - assertEquals( - "The capital of France is Paris.", - outputMessage.get("content").get(0).get("text").asText()); - assertEquals(8, outputMessage.get("usage").get("output_tokens").asInt()); - assertEquals(20, outputMessage.get("usage").get("input_tokens").asInt()); + assertNotNull(outputMessage.get("content").get(0).get("text")); + assertTrue(outputMessage.get("usage").get("output_tokens").asInt() > 0); + assertTrue(outputMessage.get("usage").get("input_tokens").asInt() > 0); // Verify time to first token Double timeToFirstToken = @@ -164,60 +111,11 @@ void testWrapAnthropic() { @Test @SneakyThrows void testWrapAnthropicStreaming() { - // Mock the Anthropic streaming API response - String streamingResponse = - """ - event: message_start - data: {"type":"message_start","message":{"id":"msg_test123","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":20,"output_tokens":1}}} - - event: content_block_start - data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} - - event: content_block_delta - data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"The"}} - - event: content_block_delta - data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" capital"}} - - event: content_block_delta - data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" of"}} - - event: content_block_delta - data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" France"}} - - event: content_block_delta - data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" is"}} - - event: content_block_delta - data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Paris"}} - - event: content_block_delta - data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"."}} - - event: content_block_stop - data: {"type":"content_block_stop","index":0} - - event: message_delta - data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":8}} - - event: message_stop - data: {"type":"message_stop"} - - """; - - wireMock.stubFor( - post(urlEqualTo("/v1/messages")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "text/event-stream") - .withBody(streamingResponse))); - - // Create Anthropic client pointing to WireMock server + // Create Anthropic client using VCR AnthropicClient anthropicClient = AnthropicOkHttpClient.builder() - .baseUrl("http://localhost:" + wireMock.getPort()) - .apiKey("test-api-key") + .baseUrl(testHarness.anthropicBaseUrl()) + .apiKey(testHarness.anthropicApiKey()) .build(); // Wrap with Braintrust instrumentation @@ -248,8 +146,7 @@ void testWrapAnthropicStreaming() { } // Verify the response - assertEquals("The capital of France is Paris.", fullResponse.toString()); - wireMock.verify(1, postRequestedFor(urlEqualTo("/v1/messages"))); + assertFalse(fullResponse.toString().isEmpty()); // Verify spans were exported var spans = testHarness.awaitExportedSpans(); @@ -262,31 +159,25 @@ void testWrapAnthropicStreaming() { assertEquals( "claude-3-5-haiku-20241022", span.getAttributes().get(AttributeKey.stringKey("gen_ai.request.model"))); - assertEquals( - "claude-3-5-haiku-20241022", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model"))); + assertNotNull(span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model"))); assertEquals( "anthropic.messages.create", span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name"))); - assertEquals( - "msg_test123", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); + assertNotNull(span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); // Verify usage metrics were captured from streaming - assertEquals( - 20L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); - assertEquals( - 8L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); + assertNotNull(span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); + assertNotNull(span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); // Verify output JSON was captured String outputJson = span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); assertNotNull(outputJson); var outputMessages = JSON_MAPPER.readTree(outputJson); - assertEquals(1, outputMessages.size()); + assertTrue(outputMessages.size() > 0); var messageZero = outputMessages.get(0); assertEquals("assistant", messageZero.get("role").asText()); - assertEquals("The capital of France is Paris.", messageZero.get("content").asText()); + assertFalse(messageZero.get("content").asText().isEmpty()); // Verify time to first token Double timeToFirstToken = diff --git a/src/test/java/dev/braintrust/instrumentation/genai/BraintrustGenAITest.java b/src/test/java/dev/braintrust/instrumentation/genai/BraintrustGenAITest.java index dd406fc..e25a74c 100644 --- a/src/test/java/dev/braintrust/instrumentation/genai/BraintrustGenAITest.java +++ b/src/test/java/dev/braintrust/instrumentation/genai/BraintrustGenAITest.java @@ -1,11 +1,8 @@ package dev.braintrust.instrumentation.genai; -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.junit.jupiter.api.Assertions.*; import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import com.google.genai.Client; import com.google.genai.types.GenerateContentConfig; import com.google.genai.types.HttpOptions; @@ -14,67 +11,31 @@ import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; public class BraintrustGenAITest { private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - @RegisterExtension - static WireMockExtension wireMock = - WireMockExtension.newInstance().options(wireMockConfig().dynamicPort()).build(); - private TestHarness testHarness; @BeforeEach void beforeEach() { testHarness = TestHarness.setup(); - wireMock.resetAll(); } @Test @SneakyThrows void testWrapGemini() { - // Mock the Gemini API response - wireMock.stubFor( - post(urlPathMatching("/v1beta/models/.*:generateContent")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody( - """ - { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "The capital of France is Paris." - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 8, - "totalTokenCount": 18 - }, - "modelVersion": "gemini-2.0-flash-lite" - } - """))); - - // Create Gemini client pointing to WireMock server + // Create Gemini client using VCR HttpOptions httpOptions = - HttpOptions.builder().baseUrl("http://localhost:" + wireMock.getPort()).build(); + HttpOptions.builder().baseUrl(testHarness.googleBaseUrl()).build(); // Wrap with Braintrust instrumentation var geminiClient = BraintrustGenAI.wrap( testHarness.openTelemetry(), - new Client.Builder().apiKey("test-api-key").httpOptions(httpOptions)); + new Client.Builder() + .apiKey(testHarness.googleApiKey()) + .httpOptions(httpOptions)); var config = GenerateContentConfig.builder().temperature(0.0f).maxOutputTokens(50).build(); @@ -84,8 +45,7 @@ void testWrapGemini() { // Verify the response assertNotNull(response); - wireMock.verify(1, postRequestedFor(urlPathMatching("/v1beta/models/.*:generateContent"))); - assertEquals("The capital of France is Paris.", response.text()); + assertNotNull(response.text()); // Verify spans were exported var spans = testHarness.awaitExportedSpans(); @@ -109,9 +69,9 @@ void testWrapGemini() { String metricsJson = span.getAttributes().get(AttributeKey.stringKey("braintrust.metrics")); assertNotNull(metricsJson, "braintrust.metrics should be set"); var metrics = JSON_MAPPER.readTree(metricsJson); - assertEquals(10, metrics.get("prompt_tokens").asInt()); - assertEquals(8, metrics.get("completion_tokens").asInt()); - assertEquals(18, metrics.get("tokens").asInt()); + assertTrue(metrics.get("prompt_tokens").asInt() > 0, "prompt_tokens should be > 0"); + assertTrue(metrics.get("completion_tokens").asInt() > 0, "completion_tokens should be > 0"); + assertTrue(metrics.get("tokens").asInt() > 0, "tokens should be > 0"); // Verify braintrust.span_attributes marks this as an LLM span String spanAttributesJson = @@ -135,62 +95,25 @@ void testWrapGemini() { assertNotNull(outputJson, "braintrust.output_json should be set"); var output = JSON_MAPPER.readTree(outputJson); assertTrue(output.has("candidates"), "output should have candidates"); - assertEquals("STOP", output.get("candidates").get(0).get("finishReason").asText()); - assertEquals( - "The capital of France is Paris.", - output.get("candidates") - .get(0) - .get("content") - .get("parts") - .get(0) - .get("text") - .asText()); + assertNotNull(output.get("candidates").get(0).get("finishReason")); + assertNotNull( + output.get("candidates").get(0).get("content").get("parts").get(0).get("text")); } @Test @SneakyThrows void testWrapGeminiAsync() { - // Mock the Gemini API response - wireMock.stubFor( - post(urlPathMatching("/v1beta/models/.*:generateContent")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody( - """ - { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "The capital of Germany is Berlin." - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 8, - "totalTokenCount": 18 - }, - "modelVersion": "gemini-2.0-flash-lite" - } - """))); - - // Create Gemini client pointing to WireMock server + // Create Gemini client using VCR HttpOptions httpOptions = - HttpOptions.builder().baseUrl("http://localhost:" + wireMock.getPort()).build(); + HttpOptions.builder().baseUrl(testHarness.googleBaseUrl()).build(); // Wrap with Braintrust instrumentation var geminiClient = BraintrustGenAI.wrap( testHarness.openTelemetry(), - new Client.Builder().apiKey("test-api-key").httpOptions(httpOptions)); + new Client.Builder() + .apiKey(testHarness.googleApiKey()) + .httpOptions(httpOptions)); var config = GenerateContentConfig.builder().temperature(0.0f).maxOutputTokens(50).build(); @@ -203,8 +126,7 @@ void testWrapGeminiAsync() { // Verify the response assertNotNull(response); - wireMock.verify(1, postRequestedFor(urlPathMatching("/v1beta/models/.*:generateContent"))); - assertEquals("The capital of Germany is Berlin.", response.text()); + assertNotNull(response.text()); // Verify spans were exported var spans = testHarness.awaitExportedSpans(); @@ -228,9 +150,9 @@ void testWrapGeminiAsync() { String metricsJson = span.getAttributes().get(AttributeKey.stringKey("braintrust.metrics")); assertNotNull(metricsJson, "braintrust.metrics should be set"); var metrics = JSON_MAPPER.readTree(metricsJson); - assertEquals(10, metrics.get("prompt_tokens").asInt()); - assertEquals(8, metrics.get("completion_tokens").asInt()); - assertEquals(18, metrics.get("tokens").asInt()); + assertTrue(metrics.get("prompt_tokens").asInt() > 0, "prompt_tokens should be > 0"); + assertTrue(metrics.get("completion_tokens").asInt() > 0, "completion_tokens should be > 0"); + assertTrue(metrics.get("tokens").asInt() > 0, "tokens should be > 0"); // Verify braintrust.span_attributes marks this as an LLM span String spanAttributesJson = @@ -254,15 +176,8 @@ void testWrapGeminiAsync() { assertNotNull(outputJson, "braintrust.output_json should be set"); var output = JSON_MAPPER.readTree(outputJson); assertTrue(output.has("candidates"), "output should have candidates"); - assertEquals("STOP", output.get("candidates").get(0).get("finishReason").asText()); - assertEquals( - "The capital of Germany is Berlin.", - output.get("candidates") - .get(0) - .get("content") - .get("parts") - .get(0) - .get("text") - .asText()); + assertNotNull(output.get("candidates").get(0).get("finishReason")); + assertNotNull( + output.get("candidates").get(0).get("content").get("parts").get(0).get("text")); } } diff --git a/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java b/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java index 9d9c983..e2555bd 100644 --- a/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java +++ b/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java @@ -1,12 +1,9 @@ package dev.braintrust.instrumentation.langchain; -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.junit.jupiter.api.Assertions.*; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import dev.braintrust.TestHarness; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.ChatModel; @@ -20,14 +17,9 @@ import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; public class BraintrustLangchainTest { - @RegisterExtension - static WireMockExtension wireMock = - WireMockExtension.newInstance().options(wireMockConfig().dynamicPort()).build(); - private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); private TestHarness testHarness; @@ -35,51 +27,17 @@ public class BraintrustLangchainTest { @BeforeEach void beforeEach() { testHarness = TestHarness.setup(); - wireMock.resetAll(); } @Test @SneakyThrows void testSyncChatCompletion() { - // Mock the OpenAI API response - wireMock.stubFor( - post(urlEqualTo("/v1/chat/completions")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody( - """ - { - "id": "chatcmpl-test123", - "object": "chat.completion", - "created": 1677652288, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "The capital of France is Paris." - }, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 20, - "completion_tokens": 8, - "total_tokens": 28 - } - } - """))); - - // Create LangChain4j client with Braintrust instrumentation ChatModel model = BraintrustLangchain.wrap( testHarness.openTelemetry(), OpenAiChatModel.builder() - .apiKey("test-api-key") - .baseUrl("http://localhost:" + wireMock.getPort() + "/v1") + .apiKey(testHarness.openAiApiKey()) + .baseUrl(testHarness.openAiBaseUrl()) .modelName("gpt-4o-mini") .temperature(0.0)); @@ -89,8 +47,7 @@ void testSyncChatCompletion() { // Verify the response assertNotNull(response); - assertEquals("The capital of France is Paris.", response.aiMessage().text()); - wireMock.verify(1, postRequestedFor(urlEqualTo("/v1/chat/completions"))); + assertNotNull(response.aiMessage().text()); // Verify spans were exported var spans = testHarness.awaitExportedSpans(); @@ -121,9 +78,10 @@ void testSyncChatCompletion() { String metricsJson = attributes.get(AttributeKey.stringKey("braintrust.metrics")); assertNotNull(metricsJson, "Metrics should be present"); JsonNode metrics = JSON_MAPPER.readTree(metricsJson); - assertEquals(28, metrics.get("tokens").asLong(), "Total tokens should be 28"); - assertEquals(20, metrics.get("prompt_tokens").asLong(), "Prompt tokens should be 20"); - assertEquals(8, metrics.get("completion_tokens").asLong(), "Completion tokens should be 8"); + assertTrue(metrics.get("tokens").asLong() > 0, "Total tokens should be > 0"); + assertTrue(metrics.get("prompt_tokens").asLong() > 0, "Prompt tokens should be > 0"); + assertTrue( + metrics.get("completion_tokens").asLong() > 0, "Completion tokens should be > 0"); assertTrue( metrics.has("time_to_first_token"), "Metrics should contain time_to_first_token"); assertTrue( @@ -146,60 +104,21 @@ void testSyncChatCompletion() { JsonNode output = JSON_MAPPER.readTree(outputJson); assertTrue(output.isArray(), "Output should be an array"); assertTrue(output.size() > 0, "Output array should not be empty"); - assertTrue( - output.get(0) - .get("message") - .get("content") - .asText() - .contains("The capital of France is Paris"), - "Output should contain the assistant response"); + assertNotNull( + output.get(0).get("message").get("content"), + "Output should contain assistant response content"); } @Test @SneakyThrows void testStreamingChatCompletion() { - // Mock the OpenAI API streaming response - String streamingResponse = - """ - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":"The"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" capital"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" of"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" France"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" is"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" Paris"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":"."},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[],"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28}} - - data: [DONE] - - """; - - wireMock.stubFor( - post(urlEqualTo("/v1/chat/completions")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "text/event-stream") - .withBody(streamingResponse))); - - // Create LangChain4j streaming client with Braintrust instrumentation + // Create LangChain4j streaming client with Braintrust instrumentation using VCR StreamingChatModel model = BraintrustLangchain.wrap( testHarness.openTelemetry(), OpenAiStreamingChatModel.builder() - .apiKey("test-api-key") - .baseUrl("http://localhost:" + wireMock.getPort() + "/v1") + .apiKey(testHarness.openAiApiKey()) + .baseUrl(testHarness.openAiBaseUrl()) .modelName("gpt-4o-mini") .temperature(0.0)); @@ -231,8 +150,7 @@ public void onError(Throwable error) { // Verify the response assertNotNull(response); - assertEquals("The capital of France is Paris.", responseBuilder.toString()); - wireMock.verify(1, postRequestedFor(urlEqualTo("/v1/chat/completions"))); + assertFalse(responseBuilder.toString().isEmpty(), "Response should not be empty"); // Verify spans were exported var spans = testHarness.awaitExportedSpans(1); @@ -264,9 +182,10 @@ public void onError(Throwable error) { String metricsJson = attributes.get(AttributeKey.stringKey("braintrust.metrics")); assertNotNull(metricsJson, "Metrics should be present"); JsonNode metrics = JSON_MAPPER.readTree(metricsJson); - assertEquals(28, metrics.get("tokens").asLong(), "Total tokens should be 28"); - assertEquals(20, metrics.get("prompt_tokens").asLong(), "Prompt tokens should be 20"); - assertEquals(8, metrics.get("completion_tokens").asLong(), "Completion tokens should be 8"); + assertTrue(metrics.get("tokens").asLong() > 0, "Total tokens should be > 0"); + assertTrue(metrics.get("prompt_tokens").asLong() > 0, "Prompt tokens should be > 0"); + assertTrue( + metrics.get("completion_tokens").asLong() > 0, "Completion tokens should be > 0"); assertTrue( metrics.has("time_to_first_token"), "Metrics should contain time_to_first_token for streaming"); @@ -291,15 +210,9 @@ public void onError(Throwable error) { assertTrue(output.isArray(), "Output should be an array"); assertTrue(output.size() > 0, "Output array should not be empty"); JsonNode choice = output.get(0); - assertTrue( - choice.get("message") - .get("content") - .asText() - .contains("The capital of France is Paris"), + assertNotNull( + choice.get("message").get("content"), "Output should contain the complete streamed response"); - assertEquals( - "stop", - choice.get("finish_reason").asText(), - "Output should have finish_reason 'stop'"); + assertNotNull(choice.get("finish_reason"), "Output should have finish_reason"); } } diff --git a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java index e131205..a6f4175 100644 --- a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java +++ b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java @@ -1,88 +1,38 @@ package dev.braintrust.instrumentation.openai; import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.junit.jupiter.api.Assertions.*; import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import com.openai.client.OpenAIClient; import com.openai.client.okhttp.OpenAIOkHttpClient; -import com.openai.core.JsonValue; import com.openai.models.ChatModel; -import com.openai.models.FunctionDefinition; -import com.openai.models.FunctionParameters; import com.openai.models.chat.completions.*; import dev.braintrust.TestHarness; -import dev.braintrust.api.BraintrustApiClient; -import dev.braintrust.prompt.BraintrustPrompt; -import dev.braintrust.trace.Base64Attachment; import io.opentelemetry.api.common.AttributeKey; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; public class BraintrustOpenAITest { - private static final ObjectMapper JSON_MAPPER = - new com.fasterxml.jackson.databind.ObjectMapper(); - - @RegisterExtension - static WireMockExtension wireMock = - WireMockExtension.newInstance().options(wireMockConfig().dynamicPort()).build(); + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); private TestHarness testHarness; @BeforeEach void beforeEach() { testHarness = TestHarness.setup(); - wireMock.resetAll(); } @Test @SneakyThrows void testWrapOpenAi() { - // Mock the OpenAI API response - wireMock.stubFor( - post(urlEqualTo("/chat/completions")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody( - """ - { - "id": "chatcmpl-test123", - "object": "chat.completion", - "created": 1677652288, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "The capital of France is Paris." - }, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 20, - "completion_tokens": 8, - "total_tokens": 28 - } - } - """))); - - // Create OpenAI client pointing to WireMock server + // Create OpenAI client using TestHarness configuration + // TestHarness automatically provides the correct base URL and API key OpenAIClient openAIClient = OpenAIOkHttpClient.builder() - .baseUrl("http://localhost:" + wireMock.getPort()) - .apiKey("test-api-key") + .baseUrl(testHarness.openAiBaseUrl()) + .apiKey(testHarness.openAiApiKey()) .build(); // Wrap with Braintrust instrumentation @@ -98,13 +48,12 @@ void testWrapOpenAi() { var response = openAIClient.chat().completions().create(request); - // Verify the response + // Verify the response (same assertions work for both modes) assertNotNull(response); - wireMock.verify(1, postRequestedFor(urlEqualTo("/chat/completions"))); - assertEquals("chatcmpl-test123", response.id()); - assertEquals( - "The capital of France is Paris.", - response.choices().get(0).message().content().get()); + assertNotNull(response.id()); + assertTrue(response.choices().get(0).message().content().isPresent()); + String content = response.choices().get(0).message().content().get(); + assertTrue(content.toLowerCase().contains("paris"), "Response should mention Paris"); // Verify spans were exported var spans = testHarness.awaitExportedSpans(); @@ -114,39 +63,19 @@ void testWrapOpenAi() { // Verify span name matches other SDKs assertEquals("Chat Completion", span.getName()); + // Verify essential span attributes assertEquals("openai", span.getAttributes().get(AttributeKey.stringKey("gen_ai.system"))); assertEquals( "gpt-4o-mini", span.getAttributes().get(AttributeKey.stringKey("gen_ai.request.model"))); - assertEquals( - "gpt-4o-mini", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model"))); - assertEquals( - "[stop]", - span.getAttributes() - .get(AttributeKey.stringArrayKey("gen_ai.response.finish_reasons")) - .toString()); + assertNotNull(span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model"))); assertEquals( "chat", span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name"))); - assertEquals( - "chatcmpl-test123", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); - assertEquals( - "[{\"role\":\"system\",\"parts\":[{\"type\":\"text\",\"content\":\"You are a" - + " helpful" - + " assistant\"}]},{\"role\":\"user\",\"parts\":[{\"type\":\"text\",\"content\":\"What" - + " is the capital of France?\"}]}]", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.input.messages"))); - assertEquals( - "project_name:" + TestHarness.defaultProjectName(), - span.getAttributes().get(AttributeKey.stringKey("braintrust.parent"))); - assertEquals( - 20L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); - assertEquals( - 8L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); - assertEquals( - 0.0, - span.getAttributes().get(AttributeKey.doubleKey("gen_ai.request.temperature"))); + assertNotNull(span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); + + // Verify usage metrics exist + assertNotNull(span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); + assertNotNull(span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); // Verify time to first token metric was captured Double timeToFirstToken = @@ -155,7 +84,7 @@ void testWrapOpenAi() { assertNotNull(timeToFirstToken, "time_to_first_token should be set"); assertTrue(timeToFirstToken >= 0.0, "time_to_first_token should be non-negative"); - // Verify Braintrust metadata was captured + // Verify Braintrust metadata assertEquals( "openai", span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata.provider"))); @@ -163,79 +92,26 @@ void testWrapOpenAi() { "gpt-4o-mini", span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata.model"))); - // Verify gen_ai.output.messages conforms to OTel spec with finish_reason + // Verify output messages structure String outputJson = span.getAttributes().get(AttributeKey.stringKey("gen_ai.output.messages")); assertNotNull(outputJson); - assertEquals( - "[{\"role\":\"assistant\",\"parts\":[{\"type\":\"text\",\"content\":\"The capital" - + " of France is Paris.\"}],\"finish_reason\":\"stop\"}]", - outputJson); - - // Verify the structure includes finish_reason and correct message content var outputMessages = JSON_MAPPER.readTree(outputJson); assertEquals(1, outputMessages.size()); var outputMessage = outputMessages.get(0); assertEquals("assistant", outputMessage.get("role").asText()); - assertEquals("stop", outputMessage.get("finish_reason").asText()); - assertTrue(outputMessage.has("parts")); - var parts = outputMessage.get("parts"); - assertEquals(1, parts.size()); - var textPart = parts.get(0); - assertEquals("text", textPart.get("type").asText()); - assertEquals("The capital of France is Paris.", textPart.get("content").asText()); - - assertEquals( - "chatcmpl-test123", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); + assertNotNull(outputMessage.get("finish_reason")); } @Test @SneakyThrows void testWrapOpenAiStreaming() { - // Mock the OpenAI API streaming response - String streamingResponse = - """ - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":"The"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" capital"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" of"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" France"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" is"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":" Paris"},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":"."},"finish_reason":null}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} - - data: {"id":"chatcmpl-test123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o-mini","choices":[],"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28}} - - data: [DONE] - - """; - - wireMock.stubFor( - post(urlEqualTo("/chat/completions")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "text/event-stream") - .withBody(streamingResponse))); - - // Create OpenAI client pointing to WireMock server OpenAIClient openAIClient = OpenAIOkHttpClient.builder() - .baseUrl("http://localhost:" + wireMock.getPort()) - .apiKey("test-api-key") + .baseUrl(testHarness.openAiBaseUrl()) + .apiKey(testHarness.openAiApiKey()) .build(); - // Wrap with Braintrust instrumentation openAIClient = BraintrustOpenAI.wrapOpenAI(testHarness.openTelemetry(), openAIClient); var request = @@ -265,430 +141,20 @@ void testWrapOpenAiStreaming() { } // Verify the response - assertEquals("The capital of France is Paris.", fullResponse.toString()); - wireMock.verify(1, postRequestedFor(urlEqualTo("/chat/completions"))); + assertFalse(fullResponse.isEmpty(), "Should have received streaming response"); + assertTrue( + fullResponse.toString().toLowerCase().contains("paris"), + "Response should mention Paris"); - // Verify spans were exported + // Verify spans var spans = testHarness.awaitExportedSpans(); assertEquals(1, spans.size()); var span = spans.get(0); - // Verify span name matches other SDKs assertEquals("Chat Completion", span.getName()); - - // Verify span attributes - assertEquals("openai", span.getAttributes().get(AttributeKey.stringKey("gen_ai.system"))); - assertEquals( - "gpt-4o-mini", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.request.model"))); - assertEquals( - "gpt-4o-mini", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model"))); - assertEquals( - "[stop]", - span.getAttributes() - .get(AttributeKey.stringArrayKey("gen_ai.response.finish_reasons")) - .toString()); - assertEquals( - "chat", span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name"))); - assertEquals( - "chatcmpl-test123", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); - - // Verify usage metrics were captured - assertEquals( - 20L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); - assertEquals( - 8L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); - - // Verify time to first token metric was captured - Double timeToFirstToken = - span.getAttributes() - .get(AttributeKey.doubleKey("braintrust.metrics.time_to_first_token")); - assertNotNull(timeToFirstToken, "time_to_first_token should be set"); - assertTrue(timeToFirstToken >= 0.0, "time_to_first_token should be non-negative"); - - // Verify Braintrust metadata was captured - assertEquals( - "openai", - span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata.provider"))); - assertEquals( - "gpt-4o-mini", - span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata.model"))); - - // Verify gen_ai.output.messages conforms to OTel spec with finish_reason - String outputJson = - span.getAttributes().get(AttributeKey.stringKey("gen_ai.output.messages")); - assertNotNull(outputJson); - - // Verify the structure includes finish_reason and correct message content - var outputMessages = JSON_MAPPER.readTree(outputJson); - assertEquals(1, outputMessages.size()); - var outputMessage = outputMessages.get(0); - assertEquals("assistant", outputMessage.get("role").asText()); - assertEquals("stop", outputMessage.get("finish_reason").asText()); - assertTrue(outputMessage.has("parts")); - var parts = outputMessage.get("parts"); - assertEquals(1, parts.size()); - var textPart = parts.get(0); - assertEquals("text", textPart.get("type").asText()); - assertEquals("The capital of France is Paris.", textPart.get("content").asText()); - } - - @Test - @SneakyThrows - void testWrapOpenAiWithImageAttachment() { - // Mock the OpenAI API response for vision request - wireMock.stubFor( - post(urlEqualTo("/chat/completions")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody( - """ - { - "id": "chatcmpl-test456", - "object": "chat.completion", - "created": 1677652288, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "This image shows the Eiffel Tower in Paris, France." - }, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 150, - "completion_tokens": 15, - "total_tokens": 165 - } - } - """))); - - // Create OpenAI client pointing to WireMock server - OpenAIClient openAIClient = - OpenAIOkHttpClient.builder() - .baseUrl("http://localhost:" + wireMock.getPort()) - .apiKey("test-api-key") - .build(); - - // Wrap with Braintrust instrumentation - openAIClient = BraintrustOpenAI.wrapOpenAI(testHarness.openTelemetry(), openAIClient); - - String imageDataUrl = - Base64Attachment.ofFile( - Base64Attachment.ContentType.IMAGE_JPEG, - "src/test/java/dev/braintrust/instrumentation/openai/travel-paris-france-poster.jpg") - .getBase64Data(); - - // Create text content part - ChatCompletionContentPartText textPart = - ChatCompletionContentPartText.builder().text("What's in this image?").build(); - ChatCompletionContentPart textContentPart = ChatCompletionContentPart.ofText(textPart); - - // Create image content part with base64-encoded image - ChatCompletionContentPartImage imagePart = - ChatCompletionContentPartImage.builder() - .imageUrl( - ChatCompletionContentPartImage.ImageUrl.builder() - // .url("https://example.com/eiffel-tower.jpg") - .url(imageDataUrl) - .detail(ChatCompletionContentPartImage.ImageUrl.Detail.HIGH) - .build()) - .build(); - ChatCompletionContentPart imageContentPart = - ChatCompletionContentPart.ofImageUrl(imagePart); - - // Create user message with both text and image - ChatCompletionUserMessageParam userMessage = - ChatCompletionUserMessageParam.builder() - .contentOfArrayOfContentParts( - Arrays.asList(textContentPart, imageContentPart)) - .build(); - - var request = - ChatCompletionCreateParams.builder() - .model(ChatModel.GPT_4O_MINI) - .addSystemMessage("You are a helpful assistant that can analyze images") - .addMessage(userMessage) - .temperature(0.0) - .build(); - - var response = openAIClient.chat().completions().create(request); - - // Verify the response - assertNotNull(response); - wireMock.verify(1, postRequestedFor(urlEqualTo("/chat/completions"))); - assertEquals("chatcmpl-test456", response.id()); - assertEquals( - "This image shows the Eiffel Tower in Paris, France.", - response.choices().get(0).message().content().get()); - - // Verify spans were exported - var spans = testHarness.awaitExportedSpans(); - assertEquals(1, spans.size()); - var span = spans.get(0); - - // Verify span attributes assertEquals("openai", span.getAttributes().get(AttributeKey.stringKey("gen_ai.system"))); - assertEquals( - "gpt-4o-mini", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.request.model"))); - assertEquals( - "gpt-4o-mini", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model"))); - assertEquals( - "[stop]", + assertNotNull( span.getAttributes() - .get(AttributeKey.stringArrayKey("gen_ai.response.finish_reasons")) - .toString()); - assertEquals( - "chat", span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name"))); - assertEquals( - "chatcmpl-test456", - span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); - - // Verify input JSON captures both text and image content - String inputJson = - span.getAttributes().get(AttributeKey.stringKey("gen_ai.input.messages")); - assertEquals( - "[{\"role\":\"system\",\"parts\":[{\"type\":\"text\",\"content\":\"You are a helpful assistant that can analyze images\"}]},{\"role\":\"user\",\"parts\":[{\"type\":\"text\",\"content\":\"What's in this image?\"},{\"type\":\"base64_attachment\",\"content\":\"%s\"}]}]" - .formatted(imageDataUrl), - inputJson); - assertNotNull(inputJson); - var inputMessages = JSON_MAPPER.readTree(inputJson); - assertEquals(2, inputMessages.size()); // system message + user message - - // Verify usage metrics - assertEquals( - 150L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); - assertEquals( - 15L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); - - // Verify output JSON - String outputJson = - span.getAttributes().get(AttributeKey.stringKey("gen_ai.output.messages")); - assertEquals( - "[{\"role\":\"assistant\",\"parts\":[{\"type\":\"text\",\"content\":\"This image" - + " shows the Eiffel Tower in Paris, France.\"}],\"finish_reason\":\"stop\"}]", - outputJson); - } - - @Test - @SneakyThrows - void testBuildChatCompletionsPrompt() { - Map promptContent = - Map.of( - "type", - "chat", - "messages", - List.of( - Map.of( - "role", "system", - "content", - "You are a kind chatbot who briefly greets people"), - Map.of( - "role", "user", - "content", "What's up my friend? My name is {{name}}"))); - - Map options = - Map.of( - "model", - "gpt-4o-mini", - "params", - Map.of("temperature", 0.1, "max_tokens", 102)); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(promptContent, options); - - BraintrustApiClient.Prompt promptObject = - new BraintrustApiClient.Prompt( - "test-id", - "test-project-id", - "test-org-id", - "kind-greeter", - "kind-greeter-test", - Optional.of("Test prompt"), - "2025-10-21T21:35:18.287Z", - promptData, - Optional.empty(), - Optional.empty()); - - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); - - Map parameters = Map.of("name", "Alice"); - ChatCompletionCreateParams renderedParams = - BraintrustOpenAI.buildChatCompletionsPrompt(prompt, parameters); - - assertEquals( - ChatCompletionCreateParams.builder() - .model(ChatModel.GPT_4O_MINI) - .temperature(0.1) - .maxTokens(102L) - .addSystemMessage("You are a kind chatbot who briefly greets people") - .addUserMessage("What's up my friend? My name is Alice") - .build(), - renderedParams); - } - - @Test - @SneakyThrows - void testWrapOpenAiWithToolCalls() { - // Mock the OpenAI API response with tool calls - wireMock.stubFor( - post(urlEqualTo("/chat/completions")) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody( - """ - { - "id": "chatcmpl-test-tool", - "object": "chat.completion", - "created": 1234567890, - "model": "gpt-4o-2024-08-06", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\\"location\\":\\"Paris, France\\"}" - } - } - ] - }, - "finish_reason": "tool_calls" - } - ], - "usage": { - "prompt_tokens": 20, - "completion_tokens": 15, - "total_tokens": 35 - } - } - """))); - - OpenAIClient client = - BraintrustOpenAI.wrapOpenAI( - testHarness.openTelemetry(), - OpenAIOkHttpClient.builder() - .baseUrl(wireMock.baseUrl()) - .apiKey("test-api-key") - .build()); - - // Make a request with tools (we'll just verify the SDK handles tool call responses - // correctly) - - ChatCompletionCreateParams params = - ChatCompletionCreateParams.builder() - .model(ChatModel.GPT_4O) - .maxTokens(500) - .temperature(0.0) - .tools( - List.of( - ChatCompletionTool.builder() - .type(JsonValue.from("function")) - .function( - FunctionDefinition.builder() - .name("get_weather") - .description( - "Get the current weather" - + " for a location") - .parameters( - FunctionParameters.builder() - .putAllAdditionalProperties( - Map.of( - "location", - JsonValue - .from( - Map - .of( - "type", - "string", - "description", - "The city" - + " and state," - + " e.g." - + " San Francisco," - + " CA")), - "unit", - JsonValue - .from( - Map - .of( - "type", - "string", - "enum", - List - .of( - "celsius", - "fahrenheit"), - "description", - "The unit" - + " of temperature")))) - .build()) - .build()) - .build())) - .addUserMessage("What is the weather like in Paris, France?") - .build(); - - ChatCompletion response = client.chat().completions().create(params); - - // Verify the response has tool calls - assertEquals(1, response.choices().size()); - var choice = response.choices().get(0); - assertEquals("tool_calls", choice.finishReason().asString()); - assertTrue(choice.message().toolCalls().isPresent()); - assertEquals(1, choice.message().toolCalls().get().size()); - var toolCall = choice.message().toolCalls().get().get(0); - assertEquals("call_abc123", toolCall.id()); - assertEquals("get_weather", toolCall.function().name()); - - // Verify spans were exported - var spans = testHarness.awaitExportedSpans(); - assertEquals(1, spans.size()); - var span = spans.get(0); - - // Verify span name - assertEquals("Chat Completion", span.getName()); - - // Verify gen_ai.output.messages conforms to OTel spec with tool calls and finish_reason - String outputJson = - span.getAttributes().get(AttributeKey.stringKey("gen_ai.output.messages")); - assertNotNull(outputJson); - - // Parse and verify the structure - var outputMessages = JSON_MAPPER.readTree(outputJson); - assertEquals(1, outputMessages.size()); - var outputMessage = outputMessages.get(0); - - // Verify role and finish_reason - assertEquals("assistant", outputMessage.get("role").asText()); - assertEquals("tool_calls", outputMessage.get("finish_reason").asText()); - - // Verify parts contain tool_call - assertTrue(outputMessage.has("parts")); - var parts = outputMessage.get("parts"); - assertEquals(1, parts.size()); - - // Verify tool call part matches OTel spec for ToolCallRequestPart - var toolCallPart = parts.get(0); - assertEquals("tool_call", toolCallPart.get("type").asText()); - assertEquals("call_abc123", toolCallPart.get("id").asText()); - assertEquals("get_weather", toolCallPart.get("name").asText()); - assertTrue(toolCallPart.get("arguments").asText().contains("Paris")); + .get(AttributeKey.doubleKey("braintrust.metrics.time_to_first_token"))); } } diff --git a/src/test/resources/cassettes/anthropic/__files/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json b/src/test/resources/cassettes/anthropic/__files/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json new file mode 100644 index 0000000..a174e92 --- /dev/null +++ b/src/test/resources/cassettes/anthropic/__files/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json @@ -0,0 +1 @@ +{"model":"claude-3-5-haiku-20241022","id":"msg_01EW7UJJzqRS5ugXCRgvUaHR","type":"message","role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard"}} \ No newline at end of file diff --git a/src/test/resources/cassettes/anthropic/__files/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.txt b/src/test/resources/cassettes/anthropic/__files/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.txt new file mode 100644 index 0000000..ac0aa3b --- /dev/null +++ b/src/test/resources/cassettes/anthropic/__files/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.txt @@ -0,0 +1,21 @@ +event: message_start +data: {"type":"message_start","message":{"model":"claude-3-5-haiku-20241022","id":"msg_01647tewtck1BoYBQHe7DP6o","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard"}} } + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"The capital of France is"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Paris."} } + +event: content_block_stop +data: {"type":"content_block_stop","index":0 } + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":10} } + +event: message_stop +data: {"type":"message_stop" } + diff --git a/src/test/resources/cassettes/anthropic/mappings/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json b/src/test/resources/cassettes/anthropic/mappings/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json new file mode 100644 index 0000000..253a39d --- /dev/null +++ b/src/test/resources/cassettes/anthropic/mappings/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json @@ -0,0 +1,49 @@ +{ + "id" : "0f8340d1-48b6-43f6-adf6-f92f304db769", + "name" : "v1_messages", + "request" : { + "url" : "/v1/messages", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"max_tokens\":50,\"messages\":[{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"claude-3-5-haiku-20241022\",\"system\":\"You are a helpful assistant\",\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : true + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json", + "headers" : { + "Date" : "Wed, 31 Dec 2025 05:49:53 GMT", + "Content-Type" : "application/json", + "anthropic-ratelimit-requests-limit" : "10000", + "anthropic-ratelimit-requests-remaining" : "9999", + "anthropic-ratelimit-requests-reset" : "2025-12-31T05:49:52Z", + "anthropic-ratelimit-input-tokens-limit" : "5000000", + "anthropic-ratelimit-input-tokens-remaining" : "5000000", + "anthropic-ratelimit-input-tokens-reset" : "2025-12-31T05:49:52Z", + "anthropic-ratelimit-output-tokens-limit" : "1000000", + "anthropic-ratelimit-output-tokens-remaining" : "1000000", + "anthropic-ratelimit-output-tokens-reset" : "2025-12-31T05:49:52Z", + "anthropic-ratelimit-tokens-limit" : "6000000", + "anthropic-ratelimit-tokens-remaining" : "6000000", + "anthropic-ratelimit-tokens-reset" : "2025-12-31T05:49:52Z", + "request-id" : "req_011CWeEvg23RAPm3zhy2ymhs", + "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", + "anthropic-organization-id" : "27796668-7351-40ac-acc4-024aee8995a5", + "Server" : "cloudflare", + "x-envoy-upstream-service-time" : "556", + "cf-cache-status" : "DYNAMIC", + "X-Robots-Tag" : "none", + "CF-RAY" : "9b677f027e99a37b-SEA" + } + }, + "uuid" : "0f8340d1-48b6-43f6-adf6-f92f304db769", + "persistent" : true, + "insertionIndex" : 1 +} \ No newline at end of file diff --git a/src/test/resources/cassettes/anthropic/mappings/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.json b/src/test/resources/cassettes/anthropic/mappings/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.json new file mode 100644 index 0000000..8af192d --- /dev/null +++ b/src/test/resources/cassettes/anthropic/mappings/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.json @@ -0,0 +1,50 @@ +{ + "id" : "db51abc2-016e-4cd2-b863-3c55bd20dc10", + "name" : "v1_messages", + "request" : { + "url" : "/v1/messages", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"max_tokens\":50,\"messages\":[{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"claude-3-5-haiku-20241022\",\"system\":\"You are a helpful assistant\",\"temperature\":0.0,\"stream\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : true + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.txt", + "headers" : { + "Date" : "Wed, 31 Dec 2025 05:49:53 GMT", + "Content-Type" : "text/event-stream; charset=utf-8", + "Cache-Control" : "no-cache", + "anthropic-ratelimit-requests-limit" : "10000", + "anthropic-ratelimit-requests-remaining" : "9999", + "anthropic-ratelimit-requests-reset" : "2025-12-31T05:49:53Z", + "anthropic-ratelimit-input-tokens-limit" : "5000000", + "anthropic-ratelimit-input-tokens-remaining" : "5000000", + "anthropic-ratelimit-input-tokens-reset" : "2025-12-31T05:49:53Z", + "anthropic-ratelimit-output-tokens-limit" : "1000000", + "anthropic-ratelimit-output-tokens-remaining" : "1000000", + "anthropic-ratelimit-output-tokens-reset" : "2025-12-31T05:49:53Z", + "anthropic-ratelimit-tokens-limit" : "6000000", + "anthropic-ratelimit-tokens-remaining" : "6000000", + "anthropic-ratelimit-tokens-reset" : "2025-12-31T05:49:53Z", + "request-id" : "req_011CWeEvkNUico2HESGkzLt2", + "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", + "anthropic-organization-id" : "27796668-7351-40ac-acc4-024aee8995a5", + "Server" : "cloudflare", + "x-envoy-upstream-service-time" : "461", + "cf-cache-status" : "DYNAMIC", + "X-Robots-Tag" : "none", + "CF-RAY" : "9b677f08d98ca3c8-SEA" + } + }, + "uuid" : "db51abc2-016e-4cd2-b863-3c55bd20dc10", + "persistent" : true, + "insertionIndex" : 2 +} \ No newline at end of file diff --git a/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json b/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json new file mode 100644 index 0000000..f797f8b --- /dev/null +++ b/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json @@ -0,0 +1,35 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The capital of Germany is Berlin.\n" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.060282785445451736 + } + ], + "usageMetadata": { + "promptTokenCount": 7, + "candidatesTokenCount": 8, + "totalTokenCount": 15, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 8 + } + ] + }, + "modelVersion": "gemini-2.0-flash-lite", + "responseId": "grlUacDXF4mP6dkP1NLR6Q0" +} diff --git a/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json b/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json new file mode 100644 index 0000000..1184072 --- /dev/null +++ b/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json @@ -0,0 +1,35 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The capital of France is **Paris**.\n" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.01313322534163793 + } + ], + "usageMetadata": { + "promptTokenCount": 7, + "candidatesTokenCount": 9, + "totalTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 9 + } + ] + }, + "modelVersion": "gemini-2.0-flash-lite", + "responseId": "g7lUaYzsB8DcqtsPgrSB2A0" +} diff --git a/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json b/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json new file mode 100644 index 0000000..7afb3fa --- /dev/null +++ b/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json @@ -0,0 +1,36 @@ +{ + "id" : "25a72094-2c7c-43cf-9084-a8cd6ab34382", + "name" : "v1beta_models_gemini-20-flash-litegeneratecontent", + "request" : { + "url" : "/v1beta/models/gemini-2.0-flash-lite:generateContent", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json; charset=UTF-8" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"contents\":[{\"parts\":[{\"text\":\"What is the capital of Germany?\"}],\"role\":\"user\"}],\"generationConfig\":{\"temperature\":0.0,\"maxOutputTokens\":50}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : true + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json", + "headers" : { + "Content-Type" : "application/json; charset=UTF-8", + "Vary" : [ "Origin", "X-Origin", "Referer" ], + "Date" : "Wed, 31 Dec 2025 05:49:54 GMT", + "Server" : "scaffolding on HTTPServer2", + "X-XSS-Protection" : "0", + "X-Frame-Options" : "SAMEORIGIN", + "X-Content-Type-Options" : "nosniff", + "Server-Timing" : "gfet4t7; dur=410", + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + } + }, + "uuid" : "25a72094-2c7c-43cf-9084-a8cd6ab34382", + "persistent" : true, + "insertionIndex" : 1 +} \ No newline at end of file diff --git a/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json b/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json new file mode 100644 index 0000000..3e74f82 --- /dev/null +++ b/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json @@ -0,0 +1,36 @@ +{ + "id" : "4559bbc0-1b54-4479-b477-bbe376bcb21b", + "name" : "v1beta_models_gemini-20-flash-litegeneratecontent", + "request" : { + "url" : "/v1beta/models/gemini-2.0-flash-lite:generateContent", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json; charset=UTF-8" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"contents\":[{\"parts\":[{\"text\":\"What is the capital of France?\"}],\"role\":\"user\"}],\"generationConfig\":{\"temperature\":0.0,\"maxOutputTokens\":50}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : true + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json", + "headers" : { + "Content-Type" : "application/json; charset=UTF-8", + "Vary" : [ "Origin", "X-Origin", "Referer" ], + "Date" : "Wed, 31 Dec 2025 05:49:55 GMT", + "Server" : "scaffolding on HTTPServer2", + "X-XSS-Protection" : "0", + "X-Frame-Options" : "SAMEORIGIN", + "X-Content-Type-Options" : "nosniff", + "Server-Timing" : "gfet4t7; dur=389", + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + } + }, + "uuid" : "4559bbc0-1b54-4479-b477-bbe376bcb21b", + "persistent" : true, + "insertionIndex" : 2 +} \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-06107a5c-8505-436f-850b-598871b122f1.txt b/src/test/resources/cassettes/openai/__files/chat_completions-06107a5c-8505-436f-850b-598871b122f1.txt new file mode 100644 index 0000000..cad7b80 --- /dev/null +++ b/src/test/resources/cassettes/openai/__files/chat_completions-06107a5c-8505-436f-850b-598871b122f1.txt @@ -0,0 +1,22 @@ +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"beUIaVqte"} + +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"dxVBy03k"} + +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" capital"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"oYL"} + +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" of"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"75W3y0x3"} + +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" France"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"1z0o"} + +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"vfaJFMjg"} + +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" Paris"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"zlAvb"} + +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"H9SDfImM9v"} + +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"JI4WH"} + +data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":7,"total_tokens":21,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"q9aPLCw6oBo"} + +data: [DONE] + diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json b/src/test/resources/cassettes/openai/__files/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json new file mode 100644 index 0000000..008b614 --- /dev/null +++ b/src/test/resources/cassettes/openai/__files/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json @@ -0,0 +1,36 @@ +{ + "id": "chatcmpl-CsjNlSbOA9QyFPqBHuOJ7rNGTbHGS", + "object": "chat.completion", + "created": 1767160197, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 14, + "completion_tokens": 7, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_c4585b5b9c" +} diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt b/src/test/resources/cassettes/openai/__files/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt new file mode 100644 index 0000000..e9253bd --- /dev/null +++ b/src/test/resources/cassettes/openai/__files/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt @@ -0,0 +1,22 @@ +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"3GIsvqZNS"} + +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"eqO4TiLK"} + +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" capital"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"jU7"} + +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" of"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"21NQowmO"} + +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" France"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"dCF0"} + +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"rQJNUmqX"} + +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" Paris"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"aJGDm"} + +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"940tkEiuxP"} + +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"kRqGD"} + +data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[],"usage":{"prompt_tokens":23,"completion_tokens":7,"total_tokens":30,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"q8zbjlCR66C"} + +data: [DONE] + diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json b/src/test/resources/cassettes/openai/__files/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json new file mode 100644 index 0000000..91ccdd5 --- /dev/null +++ b/src/test/resources/cassettes/openai/__files/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json @@ -0,0 +1,36 @@ +{ + "id": "chatcmpl-CsjNnOWy4wIVUedC2PGW1KWTecWsP", + "object": "chat.completion", + "created": 1767160199, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 23, + "completion_tokens": 7, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_29330a9688" +} diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-06107a5c-8505-436f-850b-598871b122f1.json b/src/test/resources/cassettes/openai/mappings/chat_completions-06107a5c-8505-436f-850b-598871b122f1.json new file mode 100644 index 0000000..33641c3 --- /dev/null +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-06107a5c-8505-436f-850b-598871b122f1.json @@ -0,0 +1,50 @@ +{ + "id" : "06107a5c-8505-436f-850b-598871b122f1", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"messages\" : [ {\n \"role\" : \"user\",\n \"content\" : \"What is the capital of France?\"\n } ],\n \"temperature\" : 0.0,\n \"stream\" : true,\n \"stream_options\" : {\n \"include_usage\" : true\n }\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : true + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-06107a5c-8505-436f-850b-598871b122f1.txt", + "headers" : { + "Date" : "Wed, 31 Dec 2025 05:49:56 GMT", + "Content-Type" : "text/event-stream; charset=utf-8", + "access-control-expose-headers" : "X-Request-ID", + "openai-organization" : "braintrust-data", + "openai-processing-ms" : "330", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version" : "2020-10-01", + "x-envoy-upstream-service-time" : "495", + "x-ratelimit-limit-requests" : "30000", + "x-ratelimit-limit-tokens" : "150000000", + "x-ratelimit-remaining-requests" : "29999", + "x-ratelimit-remaining-tokens" : "149999990", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-reset-tokens" : "0s", + "x-request-id" : "req_574478be32734066986a8074185255ae", + "x-openai-proxy-wasm" : "v0.1", + "cf-cache-status" : "DYNAMIC", + "Set-Cookie" : [ "__cf_bm=55HipGfK7WWBg.Vg8L4q42mu6bXkWWZ6H8lWdLjrA_U-1767160196-1.0.1.1-vqM_5zFTfWPhcNo7owkGT2ZgRcpK4VuJgaigtqiGUgEBE0o07NsPpCmiOCkwf4rn1gXKY9W.yW.fyEdmJIJXR0w9BHXnQ08BD.OCtnU9tdg; path=/; expires=Wed, 31-Dec-25 06:19:56 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=rbxezynouHlFuKWzjsrbwclTwM7KvHixfgpIOc5h2vY-1767160196378-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options" : "nosniff", + "Server" : "cloudflare", + "CF-RAY" : "9b677f17a95d7630-SEA", + "alt-svc" : "h3=\":443\"; ma=86400" + } + }, + "uuid" : "06107a5c-8505-436f-850b-598871b122f1", + "persistent" : true, + "insertionIndex" : 1 +} \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json b/src/test/resources/cassettes/openai/mappings/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json new file mode 100644 index 0000000..11e4d79 --- /dev/null +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json @@ -0,0 +1,50 @@ +{ + "id" : "657d1294-4a6e-41ca-b54a-4ab699f336bc", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"messages\" : [ {\n \"role\" : \"user\",\n \"content\" : \"What is the capital of France?\"\n } ],\n \"temperature\" : 0.0,\n \"stream\" : false\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : true + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json", + "headers" : { + "Date" : "Wed, 31 Dec 2025 05:49:58 GMT", + "Content-Type" : "application/json", + "access-control-expose-headers" : "X-Request-ID", + "openai-organization" : "braintrust-data", + "openai-processing-ms" : "790", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version" : "2020-10-01", + "x-envoy-upstream-service-time" : "803", + "x-ratelimit-limit-requests" : "30000", + "x-ratelimit-limit-tokens" : "150000000", + "x-ratelimit-remaining-requests" : "29999", + "x-ratelimit-remaining-tokens" : "149999990", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-reset-tokens" : "0s", + "x-request-id" : "req_3494dddb4ff04d71b4e04c8bdd849238", + "x-openai-proxy-wasm" : "v0.1", + "cf-cache-status" : "DYNAMIC", + "Set-Cookie" : [ "__cf_bm=xs7eo1iqAzPmMEWkRX2as3txeen9VmfNgdgv4NUA5Gk-1767160198-1.0.1.1-N41Aszum8iDWzU8bHrVs1qe0sTi_604F4U.i0IuTVFzIcaimCZU8aStvawypaumbIqZsNXtcgdBq68oJc1OOYyp5.g5O4g_Hma.Vbnu_kPs; path=/; expires=Wed, 31-Dec-25 06:19:58 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=fQPpTctWYUE6rgjuAT4gZWj9pkxzQhXeDAVonakGCw0-1767160198559-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options" : "nosniff", + "Server" : "cloudflare", + "CF-RAY" : "9b677f237ecaba4c-SEA", + "alt-svc" : "h3=\":443\"; ma=86400" + } + }, + "uuid" : "657d1294-4a6e-41ca-b54a-4ab699f336bc", + "persistent" : true, + "insertionIndex" : 2 +} \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.json b/src/test/resources/cassettes/openai/mappings/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.json new file mode 100644 index 0000000..d78f766 --- /dev/null +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.json @@ -0,0 +1,50 @@ +{ + "id" : "81d3ca7f-0bf4-4a03-b89a-105ccb33de39", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"stream_options\":{\"include_usage\":true},\"temperature\":0.0,\"stream\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : true + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt", + "headers" : { + "Date" : "Wed, 31 Dec 2025 05:49:59 GMT", + "Content-Type" : "text/event-stream; charset=utf-8", + "access-control-expose-headers" : "X-Request-ID", + "openai-organization" : "braintrust-data", + "openai-processing-ms" : "239", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version" : "2020-10-01", + "x-envoy-upstream-service-time" : "253", + "x-ratelimit-limit-requests" : "30000", + "x-ratelimit-limit-tokens" : "150000000", + "x-ratelimit-remaining-requests" : "29999", + "x-ratelimit-remaining-tokens" : "149999982", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-reset-tokens" : "0s", + "x-request-id" : "req_a5095aecf5044a0db2dcf6a3352b1384", + "x-openai-proxy-wasm" : "v0.1", + "cf-cache-status" : "DYNAMIC", + "Set-Cookie" : [ "__cf_bm=gc1.e.r5u9NPTh_gRsIeNuihBexPsRGwWt_owdt34Oo-1767160199-1.0.1.1-En_zK2nPi53SQrT1waW6yOpzYUWvD2ekMQfus0FutgB83AqAz5tppvvOfgR8HYQPWOrFKi6Ctpu_uYJ31r1Snrelh46NfhJaLNPXR4HsMpI; path=/; expires=Wed, 31-Dec-25 06:19:59 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=MBmlbrM8g3JiQhLdt5SsuDZQLoC3UhIOApeILIfdRcY-1767160199298-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options" : "nosniff", + "Server" : "cloudflare", + "CF-RAY" : "9b677f2b8fc5ba4b-SEA", + "alt-svc" : "h3=\":443\"; ma=86400" + } + }, + "uuid" : "81d3ca7f-0bf4-4a03-b89a-105ccb33de39", + "persistent" : true, + "insertionIndex" : 3 +} \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json b/src/test/resources/cassettes/openai/mappings/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json new file mode 100644 index 0000000..64e5b49 --- /dev/null +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json @@ -0,0 +1,50 @@ +{ + "id" : "9b6e1d09-eeff-4c08-a479-9a1cd5b9913a", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : true + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json", + "headers" : { + "Date" : "Wed, 31 Dec 2025 05:50:00 GMT", + "Content-Type" : "application/json", + "access-control-expose-headers" : "X-Request-ID", + "openai-organization" : "braintrust-data", + "openai-processing-ms" : "370", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version" : "2020-10-01", + "x-envoy-upstream-service-time" : "393", + "x-ratelimit-limit-requests" : "30000", + "x-ratelimit-limit-tokens" : "150000000", + "x-ratelimit-remaining-requests" : "29999", + "x-ratelimit-remaining-tokens" : "149999982", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-reset-tokens" : "0s", + "x-request-id" : "req_94fe822d95b044a58cd2eacccc013d5c", + "x-openai-proxy-wasm" : "v0.1", + "cf-cache-status" : "DYNAMIC", + "Set-Cookie" : [ "__cf_bm=juuIFMD6qt989prqR27aGApTMecE9BUuvTAD5dGhPho-1767160200-1.0.1.1-bRt_lonT2SPEwJm2Bi9EWUnJFZnsrG_3QbGf8P4OOf_f_EqfTyG0NVAh_WmPpJ1t0u039SVg1XyUd1FEjz_oE3ZlIzf3VE8oCktjReFCkKg; path=/; expires=Wed, 31-Dec-25 06:20:00 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=x5FoQ1I.0RMSh8W0juh1zeyYg.Oi1sc2Da4deU27gUM-1767160200105-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options" : "nosniff", + "Server" : "cloudflare", + "CF-RAY" : "9b677f2fc891eb6b-SEA", + "alt-svc" : "h3=\":443\"; ma=86400" + } + }, + "uuid" : "9b6e1d09-eeff-4c08-a479-9a1cd5b9913a", + "persistent" : true, + "insertionIndex" : 4 +} \ No newline at end of file