From 5123e0a5a6ff56437f10d3b78edc7237355bf5ae Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Tue, 14 Oct 2025 21:08:22 +0100 Subject: [PATCH 1/5] check model --- internal/server/web/proxy/middleware.go | 72 ++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/internal/server/web/proxy/middleware.go b/internal/server/web/proxy/middleware.go index 5a215f5..5b078c2 100644 --- a/internal/server/web/proxy/middleware.go +++ b/internal/server/web/proxy/middleware.go @@ -4,13 +4,16 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/bricks-cloud/bricksllm/internal/provider/xcustom" "io" "net/http" "strconv" "strings" "time" + "github.com/bricks-cloud/bricksllm/internal/provider/azure" + "github.com/bricks-cloud/bricksllm/internal/provider/deepinfra" + "github.com/bricks-cloud/bricksllm/internal/provider/xcustom" + "github.com/bricks-cloud/bricksllm/internal/event" "github.com/bricks-cloud/bricksllm/internal/key" "github.com/bricks-cloud/bricksllm/internal/message" @@ -938,6 +941,12 @@ func getMiddleware(cpm CustomProvidersManager, rm routeManager, pm PoliciesManag c.Abort() return } + if !isModelSupported(c.FullPath(), model) { + telemetry.Incr("bricksllm.proxy.get_middleware.model_not_supported", nil, 1) + JSON(c, http.StatusBadRequest, "[BricksLLM] model is not supported") + c.Abort() + return + } aid := c.Param("assistant_id") fid := c.Param("file_id") @@ -1332,6 +1341,67 @@ func containsPath(arr []key.PathConfig, path, method string) bool { return true } } + return false +} + +var openaiModels = map[string]struct{}{} +var anthropicModels = map[string]struct{}{} +var azureOpenAIModels = map[string]struct{}{} +var vllmModels = map[string]struct{}{} +var deepinfraModels = map[string]struct{}{} +var bedrockAnthropicModels = map[string]struct{}{} + +func init() { + // openai models + initByCostMap(openaiModels, openai.OpenAiPerThousandTokenCost) + // anthropic models + initByCostMap(anthropicModels, anthropic.AnthropicPerMillionTokenCost) + // azure openai models + initByCostMap(azureOpenAIModels, azure.AzureOpenAiPerThousandTokenCost) + // deepinfra models + initByCostMap(deepinfraModels, deepinfra.DeepinfraPerMillionTokenCost) + // TODO: check vllm and bedrock anthropic models + // cost map from settings + // maybe disable this models or... +} + +func initByCostMap(target map[string]struct{}, source map[string]map[string]float64) { + for _, m := range source { + for k, _ := range m { + target[k] = struct{}{} + } + } +} +func isModelSupported(path, model string) bool { + models := modelsMapByPath(path) + if models == nil { + return true + } + if _, ok := models[model]; ok { + return true + } return false } + +func modelsMapByPath(path string) map[string]struct{} { + if strings.HasPrefix(path, "/api/providers/openai") { + return openaiModels + } + if strings.HasPrefix(path, "/api/providers/anthropic") { + return anthropicModels + } + if strings.HasPrefix(path, "/api/providers/azure/openai") { + return azureOpenAIModels + } + if strings.HasPrefix(path, "/api/providers/vllm") { + return vllmModels + } + if strings.HasPrefix(path, "/api/providers/deepinfra") { + return deepinfraModels + } + if strings.HasPrefix(path, "/api/providers/bedrock/anthropic") { + return bedrockAnthropicModels + } + return nil +} From 2bb3d0584c9840bb72ad7152160809fbde72cee1 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Tue, 14 Oct 2025 21:53:47 +0100 Subject: [PATCH 2/5] clean --- internal/server/web/proxy/middleware.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/server/web/proxy/middleware.go b/internal/server/web/proxy/middleware.go index 5b078c2..2013b17 100644 --- a/internal/server/web/proxy/middleware.go +++ b/internal/server/web/proxy/middleware.go @@ -1347,9 +1347,7 @@ func containsPath(arr []key.PathConfig, path, method string) bool { var openaiModels = map[string]struct{}{} var anthropicModels = map[string]struct{}{} var azureOpenAIModels = map[string]struct{}{} -var vllmModels = map[string]struct{}{} var deepinfraModels = map[string]struct{}{} -var bedrockAnthropicModels = map[string]struct{}{} func init() { // openai models @@ -1360,9 +1358,6 @@ func init() { initByCostMap(azureOpenAIModels, azure.AzureOpenAiPerThousandTokenCost) // deepinfra models initByCostMap(deepinfraModels, deepinfra.DeepinfraPerMillionTokenCost) - // TODO: check vllm and bedrock anthropic models - // cost map from settings - // maybe disable this models or... } func initByCostMap(target map[string]struct{}, source map[string]map[string]float64) { @@ -1394,14 +1389,8 @@ func modelsMapByPath(path string) map[string]struct{} { if strings.HasPrefix(path, "/api/providers/azure/openai") { return azureOpenAIModels } - if strings.HasPrefix(path, "/api/providers/vllm") { - return vllmModels - } if strings.HasPrefix(path, "/api/providers/deepinfra") { return deepinfraModels } - if strings.HasPrefix(path, "/api/providers/bedrock/anthropic") { - return bedrockAnthropicModels - } return nil } From 94b25983af8e37697073ff41f45e133c0a83f0e7 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Wed, 15 Oct 2025 13:36:22 +0100 Subject: [PATCH 3/5] anthropic prices --- internal/provider/anthropic/cost.go | 53 ++++++++++++------------- internal/server/web/proxy/middleware.go | 8 +++- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/internal/provider/anthropic/cost.go b/internal/provider/anthropic/cost.go index 36680ca..8b3e0b3 100644 --- a/internal/provider/anthropic/cost.go +++ b/internal/provider/anthropic/cost.go @@ -8,21 +8,21 @@ import ( var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "prompt": { - "claude-instant": 0.8, - "claude": 8, - "claude-3-opus": 15, - "claude-3-sonnet": 3, - "claude-3.5-sonnet": 3, - "claude-3.5-haiku": 1, + "claude-sonnet-4.5": 3.0, + "claude-sonnet-4": 3.0, + "claude-3.7-sonnet": 3.0, + "claude-opus-4.1": 15.0, + "claude-opus-4": 15.0, + "claude-3.5-haiku": 0.8, "claude-3-haiku": 0.25, }, "completion": { - "claude-instant": 2.4, - "claude": 24, - "claude-3-opus": 75, - "claude-3-sonnet": 15, - "claude-3.5-sonnet": 15, - "claude-3.5-haiku": 5, + "claude-sonnet-4.5": 15.0, + "claude-sonnet-4": 15.0, + "claude-3.7-sonnet": 15.0, + "claude-opus-4.1": 75.0, + "claude-opus-4": 75.0, + "claude-3.5-haiku": 4.0, "claude-3-haiku": 1.25, }, } @@ -68,7 +68,7 @@ func (ce *CostEstimator) EstimatePromptCost(model string, tks int) (float64, err if strings.HasPrefix(model, "us") { selected = convertAmazonModelToAnthropicModel(model) } else { - selected = selectModel(model) + selected = SelectModel(model) } cost, ok := costMap[selected] @@ -80,23 +80,22 @@ func (ce *CostEstimator) EstimatePromptCost(model string, tks int) (float64, err return tksInFloat / 1000000 * cost, nil } -func selectModel(model string) string { - if strings.HasPrefix(model, "claude-3-opus") { - return "claude-3-opus" - } else if strings.HasPrefix(model, "claude-3-sonnet") { - return "claude-3-sonnet" - } else if strings.HasPrefix(model, "claude-3.5-sonnet") || strings.HasPrefix(model, "claude-3-5-sonnet") { - return "claude-3.5-sonnet" +func SelectModel(model string) string { + if strings.HasPrefix(model, "claude-sonnet-4.5") || strings.HasPrefix(model, "claude-sonnet-4-5") { + return "claude-sonnet-4.5" + } else if strings.HasPrefix(model, "claude-sonnet-4") { + return "claude-sonnet-4" + } else if strings.HasPrefix(model, "claude-3.7-sonnet") || strings.HasPrefix(model, "claude-3-7-sonnet") { + return "claude-3.7-sonnet" + } else if strings.HasPrefix(model, "claude-opus-4.1") || strings.HasPrefix(model, "claude-opus-4-1") { + return "claude-opus-4.1" + } else if strings.HasPrefix(model, "claude-opus-4") { + return "claude-opus-4" } else if strings.HasPrefix(model, "claude-3.5-haiku") || strings.HasPrefix(model, "claude-3-5-haiku") { return "claude-3.5-haiku" } else if strings.HasPrefix(model, "claude-3-haiku") { return "claude-3-haiku" - } else if strings.HasPrefix(model, "claude-instant") { - return "claude-instant" - } else if strings.HasPrefix(model, "claude") { - return "claude" } - return "" } @@ -106,7 +105,7 @@ func convertAmazonModelToAnthropicModel(model string) string { return model } - return selectModel(parts[2]) + return SelectModel(parts[2]) } func (ce *CostEstimator) EstimateCompletionCost(model string, tks int) (float64, error) { @@ -119,7 +118,7 @@ func (ce *CostEstimator) EstimateCompletionCost(model string, tks int) (float64, if strings.HasPrefix(model, "us") { selected = convertAmazonModelToAnthropicModel(model) } else { - selected = selectModel(model) + selected = SelectModel(model) } cost, ok := costMap[selected] diff --git a/internal/server/web/proxy/middleware.go b/internal/server/web/proxy/middleware.go index 2013b17..06d116d 100644 --- a/internal/server/web/proxy/middleware.go +++ b/internal/server/web/proxy/middleware.go @@ -1363,17 +1363,21 @@ func init() { func initByCostMap(target map[string]struct{}, source map[string]map[string]float64) { for _, m := range source { for k, _ := range m { - target[k] = struct{}{} + target[strings.ToLower(k)] = struct{}{} } } } func isModelSupported(path, model string) bool { + targetModel := strings.ToLower(model) + if strings.HasPrefix(path, "/api/providers/anthropic") { + targetModel = anthropic.SelectModel(targetModel) + } models := modelsMapByPath(path) if models == nil { return true } - if _, ok := models[model]; ok { + if _, ok := models[targetModel]; ok { return true } return false From f2a6d7b7fcf956a6d6c1722e079875ee0918f1e3 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Wed, 15 Oct 2025 15:59:01 +0100 Subject: [PATCH 4/5] old models --- internal/provider/anthropic/cost.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/provider/anthropic/cost.go b/internal/provider/anthropic/cost.go index 8b3e0b3..d4a6588 100644 --- a/internal/provider/anthropic/cost.go +++ b/internal/provider/anthropic/cost.go @@ -15,6 +15,9 @@ var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "claude-opus-4": 15.0, "claude-3.5-haiku": 0.8, "claude-3-haiku": 0.25, + + "claude-3.5-sonnet": 3.0, + "claude-3-opus": 15.0, }, "completion": { "claude-sonnet-4.5": 15.0, @@ -24,6 +27,9 @@ var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "claude-opus-4": 75.0, "claude-3.5-haiku": 4.0, "claude-3-haiku": 1.25, + + "claude-3.5-sonnet": 15.0, + "claude-3-opus": 75.0, }, } From b695946707a903efcf7a5e7d1d704a7440e6f4c6 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Wed, 15 Oct 2025 16:49:16 +0100 Subject: [PATCH 5/5] add to select model --- internal/provider/anthropic/cost.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/provider/anthropic/cost.go b/internal/provider/anthropic/cost.go index d4a6588..95c88be 100644 --- a/internal/provider/anthropic/cost.go +++ b/internal/provider/anthropic/cost.go @@ -101,6 +101,10 @@ func SelectModel(model string) string { return "claude-3.5-haiku" } else if strings.HasPrefix(model, "claude-3-haiku") { return "claude-3-haiku" + } else if strings.HasPrefix(model, "claude-3.5-sonnet") || strings.HasPrefix(model, "claude-3-5-sonnet") { + return "claude-3.5-sonnet" + } else if strings.HasPrefix(model, "claude-3-opus") { + return "claude-3-opus" } return "" }