Commit c36418c8 by Calcium-Ion Committed by GitHub

feat: enhance text protocol conversion and advanced custom routing (#5825)

* refactor: consolidate relay protocol converters

* refactor relayconvert text converters

* feat: refine relay converters and advanced custom routing

* refactor: enhance logging and add thought signature handling for Gemini requests

* refactor: enhance channel cache and pricing endpoint handling for advanced custom models

* feat: preserve billing usage semantics

* feat: add protocol-aware billing usage

* Delete useless files

* chore: update action versions in workflow files

* chore: update Docker action versions in workflow files

* fix: harden billing usage settlement and hot-path route matching

- estimate Gemini completion tokens locally when billable usageMetadata is
  prompt-only but output content was received (e.g. client aborts the stream
  before the final chunk), and rebuild the attached billing_usage as estimated
  so settlement does not bill zero output tokens
- guard NewClaudeMessagesBillingUsage against all-zero ClaudeUsage, matching
  the OpenAI/Gemini constructors, so a zero billing_usage cannot override a
  non-zero top-level usage during settlement
- cache compiled advanced-custom route model regexes; they run on the request
  hot path and were recompiled per request
- move the effectiveBillingUsage remap to PostTextConsumeQuota only, and
  document that calculateTextQuotaSummary expects remapped usage
- document the updatePricingLock -> channelSyncLock lock ordering that
  InitChannelCache/CacheUpdateChannel rely on, and the aux-struct pitfall in
  GeminiChatResponse.UnmarshalJSON
parent 1250fb2e
......@@ -36,7 +36,7 @@ jobs:
steps:
- name: Check out
uses: actions/checkout@v4
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: ${{ github.event_name == 'workflow_dispatch' && 0 || 1 }}
ref: ${{ github.event.inputs.tag || github.ref }}
......@@ -59,23 +59,23 @@ jobs:
echo "Building tag: ${TAG} for ${{ matrix.arch }}"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Log in to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Extract metadata (labels)
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
with:
images: calciumion/new-api
- name: Build & push
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
platforms: ${{ matrix.platform }}
......@@ -90,7 +90,7 @@ jobs:
sbom: true
- name: Install cosign
uses: sigstore/cosign-installer@v3
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Sign image with cosign
run: cosign sign --yes calciumion/new-api@${{ steps.build.outputs.digest }}
......@@ -117,7 +117,7 @@ jobs:
run: echo "TAG=${{ needs.build_single_arch.outputs.tag }}" >> $GITHUB_ENV
- name: Log in to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
......
......@@ -21,7 +21,7 @@ jobs:
contents: read
steps:
- name: Check out branch
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 1
ref: ${{ inputs.branch }}
......@@ -68,7 +68,7 @@ jobs:
id-token: write
steps:
- name: Check out branch
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 1
ref: ${{ needs.prepare.outputs.sha }}
......@@ -79,24 +79,24 @@ jobs:
echo "Publishing version: ${{ needs.prepare.outputs.version }} for ${{ matrix.arch }}"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Log in to Docker Hub
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Extract metadata (labels)
id: meta
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
with:
images: |
calciumion/new-api
- name: Build & push single-arch
id: build
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
platforms: ${{ matrix.platform }}
......@@ -111,7 +111,7 @@ jobs:
sbom: true
- name: Install cosign
uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Sign image with cosign
run: cosign sign --yes calciumion/new-api@${{ steps.build.outputs.digest }}
......@@ -133,7 +133,7 @@ jobs:
id-token: write
steps:
- name: Log in to Docker Hub
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
......@@ -153,7 +153,7 @@ jobs:
calciumion/new-api:${{ needs.prepare.outputs.version }}-arm64
- name: Install cosign
uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Sign manifests with cosign
run: |
......
......@@ -22,22 +22,22 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
- name: Setup Bun
uses: oven-sh/setup-bun@v2
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: latest
- name: Setup Node.js
uses: actions/setup-node@v4
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: '20'
- name: Setup Go
uses: actions/setup-go@v5
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with:
go-version: '>=1.25.1'
......@@ -106,7 +106,7 @@ jobs:
# - name: Upload artifacts (macOS)
# if: runner.os == 'macOS'
# uses: actions/upload-artifact@v4
# uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
# with:
# name: macos-build
# path: |
......@@ -115,7 +115,7 @@ jobs:
- name: Upload artifacts (Windows)
if: runner.os == 'Windows'
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: windows-build
path: |
......@@ -130,12 +130,12 @@ jobs:
steps:
- name: Download all artifacts
uses: actions/download-artifact@v4
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- name: Upload to Release
uses: softprops/action-gh-release@v2
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
with:
files: |
windows-build/*
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
......@@ -13,7 +13,7 @@ jobs:
pr-quality:
runs-on: ubuntu-latest
steps:
- uses: peakoss/anti-slop@v0.2.1
- uses: peakoss/anti-slop@85daca1880e9e1af197fc06ea03349daf08f4202 # v0.2.1
with:
max-failures: 4
require-description: true
......
......@@ -19,14 +19,14 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
- name: Determine Version
run: |
VERSION=$(git describe --tags)
echo "VERSION=$VERSION" >> $GITHUB_ENV
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: latest
- name: Build Frontend (default)
......@@ -48,7 +48,7 @@ jobs:
VITE_REACT_APP_VERSION=$VERSION bun run build
cd ../..
- name: Set up Go
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with:
go-version: '>=1.25.1'
- name: Build Backend (amd64)
......@@ -64,7 +64,7 @@ jobs:
run: sha256sum new-api-* > checksums-linux.txt
- name: Release
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
if: startsWith(github.ref, 'refs/tags/')
with:
files: |
......@@ -78,14 +78,14 @@ jobs:
runs-on: macos-latest
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
- name: Determine Version
run: |
VERSION=$(git describe --tags)
echo "VERSION=$VERSION" >> $GITHUB_ENV
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: latest
- name: Build Frontend (default)
......@@ -108,7 +108,7 @@ jobs:
VITE_REACT_APP_VERSION=$VERSION bun run build
cd ../..
- name: Set up Go
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with:
go-version: '>=1.25.1'
- name: Build Backend
......@@ -119,7 +119,7 @@ jobs:
run: shasum -a 256 new-api-macos-* > checksums-macos.txt
- name: Release
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
if: startsWith(github.ref, 'refs/tags/')
with:
files: |
......@@ -136,14 +136,14 @@ jobs:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
- name: Determine Version
run: |
VERSION=$(git describe --tags)
echo "VERSION=$VERSION" >> $GITHUB_ENV
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: latest
- name: Build Frontend (default)
......@@ -165,7 +165,7 @@ jobs:
VITE_REACT_APP_VERSION=$VERSION bun run build
cd ../..
- name: Set up Go
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with:
go-version: '>=1.25.1'
- name: Build Backend
......@@ -176,7 +176,7 @@ jobs:
run: sha256sum new-api-*.exe > checksums-windows.txt
- name: Release
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
if: startsWith(github.ref, 'refs/tags/')
with:
files: |
......
......@@ -131,7 +131,17 @@ func withSelfUseModeDisabled(t *testing.T) {
})
}
func decodeListModelsResponse(t *testing.T, recorder *httptest.ResponseRecorder) map[string]struct{} {
func withSelfUseModeEnabled(t *testing.T) {
t.Helper()
original := operation_setting.SelfUseModeEnabled
operation_setting.SelfUseModeEnabled = true
t.Cleanup(func() {
operation_setting.SelfUseModeEnabled = original
})
}
func decodeListModelsPayload(t *testing.T, recorder *httptest.ResponseRecorder) listModelsResponse {
t.Helper()
require.Equal(t, http.StatusOK, recorder.Code)
......@@ -139,7 +149,13 @@ func decodeListModelsResponse(t *testing.T, recorder *httptest.ResponseRecorder)
require.NoError(t, common.Unmarshal(recorder.Body.Bytes(), &payload))
require.True(t, payload.Success)
require.Equal(t, "list", payload.Object)
return payload
}
func decodeListModelsResponse(t *testing.T, recorder *httptest.ResponseRecorder) map[string]struct{} {
t.Helper()
payload := decodeListModelsPayload(t, recorder)
ids := make(map[string]struct{}, len(payload.Data))
for _, item := range payload.Data {
ids[item.Id] = struct{}{}
......@@ -255,6 +271,77 @@ func TestListModelsIncludesTieredBillingModel(t *testing.T) {
require.Empty(t, missingExprPricing.BillingExpr)
}
func TestListModelsUsesAdvancedCustomEndpointTypesFromPricingCache(t *testing.T) {
withSelfUseModeEnabled(t)
db := setupModelListControllerTestDB(t)
originalMemoryCacheEnabled := common.MemoryCacheEnabled
common.MemoryCacheEnabled = true
t.Cleanup(func() {
common.MemoryCacheEnabled = originalMemoryCacheEnabled
model.InvalidatePricingCache()
})
require.NoError(t, db.Create(&model.User{
Id: 1003,
Username: "advanced-custom-model-list-user",
Password: "password",
Group: "default",
Status: common.UserStatusEnabled,
}).Error)
channel := &model.Channel{
Id: 701,
Type: constant.ChannelTypeAdvancedCustom,
Key: "advanced-custom-key",
Status: common.ChannelStatusEnabled,
Name: "advanced-custom-channel",
Group: "default",
Models: "gemini-3.5-flash",
}
channel.SetOtherSettings(dto.ChannelOtherSettings{
AdvancedCustom: &dto.AdvancedCustomConfig{
Routes: []dto.AdvancedCustomRoute{
{
IncomingPath: "/v1/chat/completions",
UpstreamPath: "/v1/chat/completions",
},
{
IncomingPath: "/v1/responses",
UpstreamPath: "/v1beta/models/{model}:generateContent",
Converter: "openai_responses_to_gemini_generate_content",
Models: []string{"re:^gemini-"},
},
},
},
})
require.NoError(t, db.Create(channel).Error)
require.NoError(t, db.Create(&model.Ability{
Group: "default",
Model: "gemini-3.5-flash",
ChannelId: 701,
Enabled: true,
}).Error)
model.InitChannelCache()
model.GetPricing()
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/models", nil)
ctx.Set("id", 1003)
ListModels(ctx, constant.ChannelTypeOpenAI)
payload := decodeListModelsPayload(t, recorder)
require.Len(t, payload.Data, 1)
require.Equal(t, "gemini-3.5-flash", payload.Data[0].Id)
require.Equal(t, []constant.EndpointType{
constant.EndpointTypeOpenAI,
constant.EndpointTypeOpenAIResponse,
}, payload.Data[0].SupportedEndpointTypes)
}
func TestListModelsTokenLimitIncludesTieredBillingModel(t *testing.T) {
withSelfUseModeDisabled(t)
withTieredBillingConfig(t, map[string]string{
......
package dto
const (
BillingUsageSourceClaudeMessages = "claude_messages"
BillingUsageSourceGeminiChat = "gemini_chat"
BillingUsageSourceOAIChat = "oai_chat"
BillingUsageSourceOAIResponses = "oai_responses"
BillingUsageSemanticAnthropic = "anthropic"
BillingUsageSemanticGemini = "gemini"
BillingUsageSemanticOpenAI = "openai"
)
type BillingUsage struct {
Source string `json:"source,omitempty"`
Semantic string `json:"semantic,omitempty"`
Estimated bool `json:"estimated,omitempty"`
OpenAIUsage *Usage `json:"openai_usage,omitempty"`
ClaudeUsage *ClaudeUsage `json:"claude_usage,omitempty"`
GeminiUsageMetadata *GeminiUsageMetadata `json:"gemini_usage_metadata,omitempty"`
}
func NewClaudeMessagesBillingUsage(usage *ClaudeUsage) *BillingUsage {
if !HasClaudeUsageTokens(usage) {
return nil
}
return &BillingUsage{
Source: BillingUsageSourceClaudeMessages,
Semantic: BillingUsageSemanticAnthropic,
ClaudeUsage: cloneClaudeUsage(usage),
}
}
// HasClaudeUsageTokens mirrors HasOpenAIUsageTokens/HasGeminiUsageMetadataTokens:
// an all-zero ClaudeUsage must not become a BillingUsage, otherwise it would take
// precedence during settlement and zero out a non-zero top-level usage.
func HasClaudeUsageTokens(usage *ClaudeUsage) bool {
if usage == nil {
return false
}
if usage.InputTokens != 0 ||
usage.OutputTokens != 0 ||
usage.CacheCreationInputTokens != 0 ||
usage.CacheReadInputTokens != 0 ||
usage.ClaudeCacheCreation5mTokens != 0 ||
usage.ClaudeCacheCreation1hTokens != 0 {
return true
}
if usage.CacheCreation != nil &&
(usage.CacheCreation.Ephemeral5mInputTokens != 0 || usage.CacheCreation.Ephemeral1hInputTokens != 0) {
return true
}
return false
}
func NewOpenAIChatBillingUsage(usage *Usage) *BillingUsage {
return newOpenAIBillingUsage(BillingUsageSourceOAIChat, usage)
}
func NewOpenAIResponsesBillingUsage(usage *Usage) *BillingUsage {
return newOpenAIBillingUsage(BillingUsageSourceOAIResponses, usage)
}
func newOpenAIBillingUsage(source string, usage *Usage) *BillingUsage {
if !HasOpenAIUsageTokens(usage) {
return nil
}
return &BillingUsage{
Source: source,
Semantic: BillingUsageSemanticOpenAI,
OpenAIUsage: cloneOpenAIUsage(usage),
}
}
func HasOpenAIUsageTokens(usage *Usage) bool {
if usage == nil {
return false
}
if usage.PromptTokens != 0 ||
usage.CompletionTokens != 0 ||
usage.TotalTokens != 0 ||
usage.InputTokens != 0 ||
usage.OutputTokens != 0 ||
usage.PromptCacheHitTokens != 0 ||
usage.ClaudeCacheCreation5mTokens != 0 ||
usage.ClaudeCacheCreation1hTokens != 0 {
return true
}
if usage.PromptTokensDetails.CachedTokens != 0 ||
usage.PromptTokensDetails.CachedCreationTokens != 0 ||
usage.PromptTokensDetails.TextTokens != 0 ||
usage.PromptTokensDetails.ImageTokens != 0 ||
usage.PromptTokensDetails.AudioTokens != 0 {
return true
}
if usage.CompletionTokenDetails.ReasoningTokens != 0 ||
usage.CompletionTokenDetails.TextTokens != 0 ||
usage.CompletionTokenDetails.ImageTokens != 0 ||
usage.CompletionTokenDetails.AudioTokens != 0 {
return true
}
return usage.InputTokensDetails != nil
}
func NewGeminiChatBillingUsage(metadata *GeminiUsageMetadata) *BillingUsage {
return newGeminiChatBillingUsage(metadata, false)
}
func NewEstimatedGeminiChatBillingUsage(usage *Usage) *BillingUsage {
if usage == nil {
return nil
}
totalTokens := usage.TotalTokens
if totalTokens == 0 {
totalTokens = usage.PromptTokens + usage.CompletionTokens
}
return newGeminiChatBillingUsage(&GeminiUsageMetadata{
PromptTokenCount: usage.PromptTokens,
CandidatesTokenCount: usage.CompletionTokens,
TotalTokenCount: totalTokens,
}, true)
}
func newGeminiChatBillingUsage(metadata *GeminiUsageMetadata, estimated bool) *BillingUsage {
if !HasGeminiUsageMetadataTokens(metadata) {
return nil
}
usageMetadata := cloneGeminiUsageMetadata(*metadata)
return &BillingUsage{
Source: BillingUsageSourceGeminiChat,
Semantic: BillingUsageSemanticGemini,
Estimated: estimated,
GeminiUsageMetadata: &usageMetadata,
}
}
func CloneBillingUsage(usage *BillingUsage) *BillingUsage {
if usage == nil {
return nil
}
clone := *usage
clone.OpenAIUsage = cloneOpenAIUsage(usage.OpenAIUsage)
clone.ClaudeUsage = cloneClaudeUsage(usage.ClaudeUsage)
if usage.GeminiUsageMetadata != nil {
metadata := cloneGeminiUsageMetadata(*usage.GeminiUsageMetadata)
clone.GeminiUsageMetadata = &metadata
}
return &clone
}
func cloneOpenAIUsage(usage *Usage) *Usage {
if usage == nil {
return nil
}
clone := *usage
clone.BillingUsage = nil
if usage.InputTokensDetails != nil {
inputTokensDetails := *usage.InputTokensDetails
clone.InputTokensDetails = &inputTokensDetails
}
return &clone
}
func cloneClaudeUsage(usage *ClaudeUsage) *ClaudeUsage {
if usage == nil {
return nil
}
clone := *usage
clone.BillingUsage = nil
if usage.CacheCreation != nil {
cacheCreation := *usage.CacheCreation
clone.CacheCreation = &cacheCreation
}
if usage.ServerToolUse != nil {
serverToolUse := *usage.ServerToolUse
clone.ServerToolUse = &serverToolUse
}
return &clone
}
func cloneGeminiUsageMetadata(metadata GeminiUsageMetadata) GeminiUsageMetadata {
metadata.PromptTokensDetails = append([]GeminiPromptTokensDetails{}, metadata.PromptTokensDetails...)
metadata.ToolUsePromptTokensDetails = append([]GeminiPromptTokensDetails{}, metadata.ToolUsePromptTokensDetails...)
metadata.CandidatesTokensDetails = append([]GeminiPromptTokensDetails{}, metadata.CandidatesTokensDetails...)
metadata.BillingUsage = nil
return metadata
}
func HasGeminiUsageMetadataTokens(metadata *GeminiUsageMetadata) bool {
if metadata == nil {
return false
}
if metadata.PromptTokenCount != 0 ||
metadata.ToolUsePromptTokenCount != 0 ||
metadata.CandidatesTokenCount != 0 ||
metadata.TotalTokenCount != 0 ||
metadata.ThoughtsTokenCount != 0 ||
metadata.CachedContentTokenCount != 0 {
return true
}
for _, detail := range metadata.PromptTokensDetails {
if detail.TokenCount != 0 {
return true
}
}
for _, detail := range metadata.ToolUsePromptTokensDetails {
if detail.TokenCount != 0 {
return true
}
}
for _, detail := range metadata.CandidatesTokensDetails {
if detail.TokenCount != 0 {
return true
}
}
return false
}
package dto
import (
"testing"
"github.com/QuantumNous/new-api/common"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewGeminiChatBillingUsageRequiresTokenContent(t *testing.T) {
require.Nil(t, NewGeminiChatBillingUsage(nil))
require.Nil(t, NewGeminiChatBillingUsage(&GeminiUsageMetadata{}))
billingUsage := NewGeminiChatBillingUsage(&GeminiUsageMetadata{PromptTokenCount: 1})
require.NotNil(t, billingUsage)
require.NotNil(t, billingUsage.GeminiUsageMetadata)
assert.Equal(t, BillingUsageSourceGeminiChat, billingUsage.Source)
assert.Equal(t, BillingUsageSemanticGemini, billingUsage.Semantic)
assert.False(t, billingUsage.Estimated)
}
func TestNewClaudeMessagesBillingUsageRequiresTokenContent(t *testing.T) {
require.Nil(t, NewClaudeMessagesBillingUsage(nil))
require.Nil(t, NewClaudeMessagesBillingUsage(&ClaudeUsage{}))
require.Nil(t, NewClaudeMessagesBillingUsage(&ClaudeUsage{CacheCreation: &ClaudeCacheCreationUsage{}}))
billingUsage := NewClaudeMessagesBillingUsage(&ClaudeUsage{InputTokens: 1})
require.NotNil(t, billingUsage)
require.NotNil(t, billingUsage.ClaudeUsage)
assert.Equal(t, BillingUsageSourceClaudeMessages, billingUsage.Source)
assert.Equal(t, BillingUsageSemanticAnthropic, billingUsage.Semantic)
cacheOnly := NewClaudeMessagesBillingUsage(&ClaudeUsage{
CacheCreation: &ClaudeCacheCreationUsage{Ephemeral5mInputTokens: 4},
})
require.NotNil(t, cacheOnly)
}
func TestNewOpenAIChatBillingUsageRequiresTokenContent(t *testing.T) {
require.Nil(t, NewOpenAIChatBillingUsage(nil))
require.Nil(t, NewOpenAIChatBillingUsage(&Usage{}))
billingUsage := NewOpenAIChatBillingUsage(&Usage{PromptTokens: 1})
require.NotNil(t, billingUsage)
require.NotNil(t, billingUsage.OpenAIUsage)
assert.Equal(t, BillingUsageSourceOAIChat, billingUsage.Source)
assert.Equal(t, BillingUsageSemanticOpenAI, billingUsage.Semantic)
assert.Equal(t, 1, billingUsage.OpenAIUsage.PromptTokens)
}
func TestNewEstimatedGeminiChatBillingUsage(t *testing.T) {
billingUsage := NewEstimatedGeminiChatBillingUsage(&Usage{
PromptTokens: 11,
CompletionTokens: 7,
})
require.NotNil(t, billingUsage)
require.NotNil(t, billingUsage.GeminiUsageMetadata)
assert.True(t, billingUsage.Estimated)
assert.Equal(t, 11, billingUsage.GeminiUsageMetadata.PromptTokenCount)
assert.Equal(t, 7, billingUsage.GeminiUsageMetadata.CandidatesTokenCount)
assert.Equal(t, 18, billingUsage.GeminiUsageMetadata.TotalTokenCount)
}
func TestBillingUsageJSONUsesProtocolNamedFields(t *testing.T) {
billingUsage := &BillingUsage{
OpenAIUsage: &Usage{PromptTokens: 1, BillingUsage: NewClaudeMessagesBillingUsage(&ClaudeUsage{InputTokens: 9})},
ClaudeUsage: &ClaudeUsage{InputTokens: 2, BillingUsage: NewOpenAIChatBillingUsage(&Usage{PromptTokens: 8})},
GeminiUsageMetadata: &GeminiUsageMetadata{PromptTokenCount: 3, BillingUsage: NewOpenAIChatBillingUsage(&Usage{PromptTokens: 7})},
}
data, err := common.Marshal(billingUsage)
require.NoError(t, err)
assert.Contains(t, string(data), `"openai_usage"`)
assert.Contains(t, string(data), `"claude_usage"`)
assert.Contains(t, string(data), `"gemini_usage_metadata"`)
assert.NotContains(t, string(data), `"usage":`)
assert.NotContains(t, string(data), `"usage_metadata"`)
clone := CloneBillingUsage(billingUsage)
require.NotNil(t, clone.OpenAIUsage)
require.NotNil(t, clone.ClaudeUsage)
require.NotNil(t, clone.GeminiUsageMetadata)
assert.Nil(t, clone.OpenAIUsage.BillingUsage)
assert.Nil(t, clone.ClaudeUsage.BillingUsage)
assert.Nil(t, clone.GeminiUsageMetadata.BillingUsage)
}
......@@ -564,6 +564,7 @@ type ClaudeUsage struct {
ClaudeCacheCreation5mTokens int `json:"claude_cache_creation_5_m_tokens"`
ClaudeCacheCreation1hTokens int `json:"claude_cache_creation_1_h_tokens"`
ServerToolUse *ClaudeServerToolUse `json:"server_tool_use,omitempty"`
BillingUsage *BillingUsage `json:"billing_usage,omitempty"`
}
type ClaudeCacheCreationUsage struct {
......
......@@ -44,9 +44,9 @@ func (r *GeminiChatRequest) UnmarshalJSON(data []byte) error {
}
type ToolConfig struct {
FunctionCallingConfig *FunctionCallingConfig `json:"functionCallingConfig,omitempty"`
RetrievalConfig *RetrievalConfig `json:"retrievalConfig,omitempty"`
IncludeServerSideToolInvocations *bool `json:"includeServerSideToolInvocations,omitempty"`
FunctionCallingConfig *FunctionCallingConfig `json:"functionCallingConfig,omitempty"`
RetrievalConfig *RetrievalConfig `json:"retrievalConfig,omitempty"`
IncludeServerSideToolInvocations *bool `json:"includeServerSideToolInvocations,omitempty"`
}
type FunctionCallingConfig struct {
......@@ -455,9 +455,46 @@ type GeminiChatPromptFeedback struct {
}
type GeminiChatResponse struct {
Candidates []GeminiChatCandidate `json:"candidates"`
PromptFeedback *GeminiChatPromptFeedback `json:"promptFeedback,omitempty"`
UsageMetadata GeminiUsageMetadata `json:"usageMetadata"`
Candidates []GeminiChatCandidate `json:"candidates"`
PromptFeedback *GeminiChatPromptFeedback `json:"promptFeedback,omitempty"`
UsageMetadata GeminiUsageMetadata `json:"usageMetadata"`
HasUsageMetadata bool `json:"-"`
}
// UnmarshalJSON records whether Gemini returned usageMetadata while preserving
// the historical wire shape that always marshals the usageMetadata field.
//
// IMPORTANT: aux shadows GeminiChatResponse. Any field added to
// GeminiChatResponse must also be added to aux (and copied below), otherwise it
// is silently dropped during unmarshal.
func (r *GeminiChatResponse) UnmarshalJSON(data []byte) error {
var aux struct {
Candidates []GeminiChatCandidate `json:"candidates"`
PromptFeedback *GeminiChatPromptFeedback `json:"promptFeedback,omitempty"`
UsageMetadata *GeminiUsageMetadata `json:"usageMetadata"`
}
if err := common.Unmarshal(data, &aux); err != nil {
return err
}
r.Candidates = aux.Candidates
r.PromptFeedback = aux.PromptFeedback
r.HasUsageMetadata = aux.UsageMetadata != nil
if aux.UsageMetadata != nil {
r.UsageMetadata = *aux.UsageMetadata
} else {
r.UsageMetadata = GeminiUsageMetadata{}
}
return nil
}
func (r *GeminiChatResponse) GetUsageMetadata() *GeminiUsageMetadata {
if r == nil {
return nil
}
if r.HasUsageMetadata || HasGeminiUsageMetadataTokens(&r.UsageMetadata) {
return &r.UsageMetadata
}
return nil
}
type GeminiUsageMetadata struct {
......@@ -470,6 +507,7 @@ type GeminiUsageMetadata struct {
PromptTokensDetails []GeminiPromptTokensDetails `json:"promptTokensDetails"`
ToolUsePromptTokensDetails []GeminiPromptTokensDetails `json:"toolUsePromptTokensDetails"`
CandidatesTokensDetails []GeminiPromptTokensDetails `json:"candidatesTokensDetails"`
BillingUsage *BillingUsage `json:"billing_usage,omitempty"`
}
type GeminiPromptTokensDetails struct {
......
package dto
import (
"testing"
"github.com/QuantumNous/new-api/common"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGeminiChatResponseUsageMetadataPresence(t *testing.T) {
var missing GeminiChatResponse
require.NoError(t, common.Unmarshal([]byte(`{"candidates":[]}`), &missing))
assert.False(t, missing.HasUsageMetadata)
assert.Nil(t, missing.GetUsageMetadata())
var empty GeminiChatResponse
require.NoError(t, common.Unmarshal([]byte(`{"candidates":[],"usageMetadata":{}}`), &empty))
assert.True(t, empty.HasUsageMetadata)
require.NotNil(t, empty.GetUsageMetadata())
assert.False(t, HasGeminiUsageMetadataTokens(empty.GetUsageMetadata()))
var populated GeminiChatResponse
require.NoError(t, common.Unmarshal([]byte(`{"candidates":[],"usageMetadata":{"promptTokenCount":3}}`), &populated))
assert.True(t, populated.HasUsageMetadata)
require.NotNil(t, populated.GetUsageMetadata())
assert.True(t, HasGeminiUsageMetadataTokens(populated.GetUsageMetadata()))
}
func TestGeminiChatResponseMarshalKeepsUsageMetadataField(t *testing.T) {
data, err := common.Marshal(GeminiChatResponse{})
require.NoError(t, err)
assert.Contains(t, string(data), `"usageMetadata"`)
}
......@@ -221,12 +221,13 @@ type CompletionsStreamResponse struct {
}
type Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
PromptCacheHitTokens int `json:"prompt_cache_hit_tokens,omitempty"`
UsageSemantic string `json:"usage_semantic,omitempty"`
UsageSource string `json:"usage_source,omitempty"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
PromptCacheHitTokens int `json:"prompt_cache_hit_tokens,omitempty"`
UsageSemantic string `json:"usage_semantic,omitempty"`
UsageSource string `json:"usage_source,omitempty"`
BillingUsage *BillingUsage `json:"billing_usage,omitempty"`
PromptTokensDetails InputTokenDetails `json:"prompt_tokens_details"`
CompletionTokenDetails OutputTokenDetails `json:"completion_tokens_details"`
......
......@@ -102,6 +102,10 @@ func main() {
go model.SyncChannelCache(common.SyncFrequency)
}
// Warm pricing after channel cache initialization so Advanced Custom
// endpoint inference can read cached route settings on first request.
model.GetPricing()
// 热更新配置
go model.SyncOptions(common.SyncFrequency)
......@@ -330,9 +334,6 @@ func InitResources() error {
// 清理旧的磁盘缓存文件
common.CleanupOldCacheFiles()
// 初始化模型
model.GetPricing()
// Initialize SQL Database
err = model.InitLogDB()
if err != nil {
......
......@@ -105,7 +105,7 @@ func Distribute() func(c *gin.Context) {
affinityUsable := false
preferred, err := model.CacheGetChannel(preferredChannelID)
if err == nil && preferred != nil && preferred.Status == common.ChannelStatusEnabled &&
channelSupportsRequestPath(preferred, c.Request.URL.Path) {
channelSupportsRequestPath(preferred, c.Request.URL.Path, modelRequest.Model) {
if usingGroup == "auto" {
userGroup := common.GetContextKeyString(c, constant.ContextKeyUserGroup)
autoGroups := service.GetUserAutoGroup(userGroup)
......@@ -172,7 +172,7 @@ func Distribute() func(c *gin.Context) {
// channelSupportsRequestPath reports whether a channel can serve the request path.
// Only Advanced Custom (type 58) channels are path-checked; all other channel types
// always pass. A type-58 channel is usable only when one of its routes matches.
func channelSupportsRequestPath(channel *model.Channel, requestPath string) bool {
func channelSupportsRequestPath(channel *model.Channel, requestPath string, requestModel string) bool {
if channel == nil {
return false
}
......@@ -180,7 +180,7 @@ func channelSupportsRequestPath(channel *model.Channel, requestPath string) bool
return true
}
config := channel.GetOtherSettings().AdvancedCustom
return config != nil && config.SupportsPath(requestPath)
return config != nil && config.SupportsPathForModel(requestPath, requestModel)
}
// getModelFromRequest 从请求中读取模型信息
......
......@@ -121,7 +121,7 @@ func GetChannel(group string, model string, retry int, requestPath string) (*Cha
if err != nil {
return nil, err
}
abilities = filterAbilitiesByRequestPath(abilities, requestPath)
abilities = filterAbilitiesByRequestPathAndModel(abilities, requestPath, model)
channel := Channel{}
if len(abilities) > 0 {
// Randomly choose one
......@@ -146,11 +146,12 @@ func GetChannel(group string, model string, retry int, requestPath string) (*Cha
return &channel, err
}
// filterAbilitiesByRequestPath restricts candidates by request path for the DB
// (non-memory-cache) selection path. Only Advanced Custom (type 58) channels are
// path-checked: kept only when one of their routes matches requestPath; all other
// channel types always pass. When requestPath is empty, filtering is skipped.
func filterAbilitiesByRequestPath(abilities []Ability, requestPath string) []Ability {
// filterAbilitiesByRequestPathAndModel restricts candidates by request path and
// model for the DB (non-memory-cache) selection path. Only Advanced Custom
// (type 58) channels are path-checked: kept only when one of their routes matches
// requestPath and model; all other channel types always pass. When requestPath is
// empty, filtering is skipped.
func filterAbilitiesByRequestPathAndModel(abilities []Ability, requestPath string, model string) []Ability {
if requestPath == "" || len(abilities) == 0 {
return abilities
}
......@@ -185,7 +186,7 @@ func filterAbilitiesByRequestPath(abilities []Ability, requestPath string) []Abi
filtered = append(filtered, ability)
continue
}
if config != nil && config.SupportsPath(requestPath) {
if config != nil && config.SupportsPathForModel(requestPath, model) {
filtered = append(filtered, ability)
}
}
......
......@@ -25,6 +25,7 @@ var channelSyncLock sync.RWMutex
func InitChannelCache() {
if !common.MemoryCacheEnabled {
InvalidatePricingCache()
return
}
newChannelId2channel := make(map[int]*Channel)
......@@ -94,6 +95,11 @@ func InitChannelCache() {
channelsIDM = newChannelId2channel
channel2advancedCustomConfig = newChannel2advancedCustomConfig
channelSyncLock.Unlock()
// Lock ordering: InvalidatePricingCache acquires updatePricingLock, and
// GetPricing (holding updatePricingLock) nests channelSyncLock.RLock via
// loadPricingAdvancedCustomConfigs. channelSyncLock MUST be released before
// invalidating the pricing cache, otherwise the reversed order deadlocks.
InvalidatePricingCache()
common.SysLog("channels synced from database")
}
......@@ -115,12 +121,12 @@ func GetRandomSatisfiedChannel(group string, model string, retry int, requestPat
defer channelSyncLock.RUnlock()
// First, try to find channels with the exact model name.
channels := filterChannelsByRequestPath(group2model2channels[group][model], requestPath)
channels := filterChannelsByRequestPathAndModel(group2model2channels[group][model], requestPath, model)
// If no channels found, try to find channels with the normalized model name.
if len(channels) == 0 {
normalizedModel := ratio_setting.FormatMatchingModelName(model)
channels = filterChannelsByRequestPath(group2model2channels[group][normalizedModel], requestPath)
channels = filterChannelsByRequestPathAndModel(group2model2channels[group][normalizedModel], requestPath, model)
}
if len(channels) == 0 {
......@@ -202,12 +208,12 @@ func GetRandomSatisfiedChannel(group string, model string, retry int, requestPat
return nil, errors.New("channel not found")
}
// filterChannelsByRequestPath restricts candidates by request path. Only Advanced
// Custom (type 58) channels are path-checked: they are kept only when one of their
// configured routes matches requestPath. All other channel types always pass.
// When requestPath is empty (non-relay callers) filtering is skipped.
// filterChannelsByRequestPathAndModel restricts candidates by request path and
// model. Only Advanced Custom (type 58) channels are path-checked: they are kept
// only when one of their configured routes matches requestPath and model. All
// other channel types always pass. When requestPath is empty, filtering is skipped.
// Caller must hold channelSyncLock (read lock). The cached slice is never mutated.
func filterChannelsByRequestPath(channels []int, requestPath string) []int {
func filterChannelsByRequestPathAndModel(channels []int, requestPath string, model string) []int {
if requestPath == "" || len(channels) == 0 {
return channels
}
......@@ -223,7 +229,7 @@ func filterChannelsByRequestPath(channels []int, requestPath string) []int {
filtered = append(filtered, channelId)
continue
}
if config := channel2advancedCustomConfig[channelId]; config != nil && config.SupportsPath(requestPath) {
if config := channel2advancedCustomConfig[channelId]; config != nil && config.SupportsPathForModel(requestPath, model) {
filtered = append(filtered, channelId)
}
}
......@@ -292,8 +298,8 @@ func CacheUpdateChannel(channel *Channel) {
return
}
channelSyncLock.Lock()
defer channelSyncLock.Unlock()
if channel == nil {
channelSyncLock.Unlock()
return
}
......@@ -304,5 +310,20 @@ func CacheUpdateChannel(channel *Channel) {
logger.LogDebug(nil, "CacheUpdateChannel before: id=%d, name=%s, status=%d, polling_index=%d", channel.Id, channel.Name, channel.Status, oldChannel.ChannelInfo.MultiKeyPollingIndex)
}
channelsIDM[channel.Id] = channel
if channel2advancedCustomConfig == nil {
channel2advancedCustomConfig = make(map[int]*dto.AdvancedCustomConfig)
}
delete(channel2advancedCustomConfig, channel.Id)
if channel.Type == constant.ChannelTypeAdvancedCustom {
if config := channel.GetOtherSettings().AdvancedCustom; config != nil {
channel2advancedCustomConfig[channel.Id] = config
}
}
logger.LogDebug(nil, "CacheUpdateChannel after: id=%d, name=%s, status=%d, polling_index=%d", channel.Id, channel.Name, channel.Status, channel.ChannelInfo.MultiKeyPollingIndex)
// Lock ordering: do NOT hold channelSyncLock while calling
// InvalidatePricingCache. GetPricing acquires updatePricingLock first and then
// channelSyncLock.RLock (via loadPricingAdvancedCustomConfigs); acquiring
// updatePricingLock while holding channelSyncLock would be an AB-BA deadlock.
channelSyncLock.Unlock()
InvalidatePricingCache()
}
package model
import (
"encoding/json"
"fmt"
"strings"
......@@ -10,6 +9,7 @@ import (
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/constant"
"github.com/QuantumNous/new-api/dto"
"github.com/QuantumNous/new-api/setting/billing_setting"
"github.com/QuantumNous/new-api/setting/ratio_setting"
"github.com/QuantumNous/new-api/types"
......@@ -107,6 +107,76 @@ func GetModelSupportEndpointTypes(model string) []constant.EndpointType {
return make([]constant.EndpointType, 0)
}
func getPricingEndpointTypesForAbility(ability AbilityWithChannel, advancedCustomConfigs map[int]*dto.AdvancedCustomConfig) []constant.EndpointType {
if ability.ChannelType != constant.ChannelTypeAdvancedCustom {
return common.GetEndpointTypesByChannelType(ability.ChannelType, ability.Model)
}
if config := advancedCustomConfigs[ability.ChannelId]; config != nil {
return config.SupportedEndpointTypesForModel(ability.Model)
}
return common.GetEndpointTypesByChannelType(ability.ChannelType, ability.Model)
}
// loadPricingAdvancedCustomConfigs runs inside updatePricing while
// updatePricingLock is held, and nests channelSyncLock.RLock. This defines the
// global lock order updatePricingLock -> channelSyncLock: any code path holding
// channelSyncLock must release it before touching the pricing cache (see
// InitChannelCache / CacheUpdateChannel), otherwise it deadlocks.
// The returned configs are pointers shared with the channel cache; they are
// replaced wholesale on update and never mutated in place, so reading them after
// RUnlock is safe.
func loadPricingAdvancedCustomConfigs(enableAbilities []AbilityWithChannel) map[int]*dto.AdvancedCustomConfig {
channelIDs := make([]int, 0)
seen := make(map[int]struct{})
for _, ability := range enableAbilities {
if ability.ChannelType != constant.ChannelTypeAdvancedCustom {
continue
}
if _, exists := seen[ability.ChannelId]; exists {
continue
}
seen[ability.ChannelId] = struct{}{}
channelIDs = append(channelIDs, ability.ChannelId)
}
if len(channelIDs) == 0 {
return nil
}
configs := make(map[int]*dto.AdvancedCustomConfig, len(channelIDs))
if common.MemoryCacheEnabled {
channelSyncLock.RLock()
defer channelSyncLock.RUnlock()
for _, channelID := range channelIDs {
if config := channel2advancedCustomConfig[channelID]; config != nil {
configs[channelID] = config
}
}
return configs
}
for _, channelID := range channelIDs {
channel, err := CacheGetChannel(channelID)
if err != nil {
common.SysLog(fmt.Sprintf("load advanced custom channel settings error: channel_id=%d, error=%v", channelID, err))
continue
}
if channel.Type != constant.ChannelTypeAdvancedCustom {
continue
}
if config := channel.GetOtherSettings().AdvancedCustom; config != nil {
configs[channelID] = config
}
}
return configs
}
func appendPricingEndpoint(endpoints []string, endpoint string) []string {
if endpoint == "" || common.StringsContains(endpoints, endpoint) {
return endpoints
}
return append(endpoints, endpoint)
}
func updatePricing() {
//modelRatios := common.GetModelRatios()
enableAbilities, err := GetAllEnableAbilityWithChannels()
......@@ -201,11 +271,12 @@ func updatePricing() {
//这里使用切片而不是Set,因为一个模型可能支持多个端点类型,并且第一个端点是优先使用端点
modelSupportEndpointsStr := make(map[string][]string)
advancedCustomConfigs := loadPricingAdvancedCustomConfigs(enableAbilities)
// 先根据已有能力填充原生端点
for _, ability := range enableAbilities {
endpoints := modelSupportEndpointsStr[ability.Model]
channelTypes := common.GetEndpointTypesByChannelType(ability.ChannelType, ability.Model)
channelTypes := getPricingEndpointTypesForAbility(ability, advancedCustomConfigs)
for _, channelType := range channelTypes {
if !common.StringsContains(endpoints, string(channelType)) {
endpoints = append(endpoints, string(channelType))
......@@ -214,20 +285,18 @@ func updatePricing() {
modelSupportEndpointsStr[ability.Model] = endpoints
}
// 再补充模型自定义端点:若配置有效则替换默认端点,不做合并
// 再补充模型自定义端点:若配置有效则追加到已有推断,不再裁剪渠道真实能力
for modelName, meta := range metaMap {
if strings.TrimSpace(meta.Endpoints) == "" {
continue
}
var raw map[string]interface{}
if err := json.Unmarshal([]byte(meta.Endpoints), &raw); err == nil {
endpoints := make([]string, 0, len(raw))
if err := common.Unmarshal([]byte(meta.Endpoints), &raw); err == nil {
endpoints := modelSupportEndpointsStr[modelName]
for k, v := range raw {
switch v.(type) {
case string, map[string]interface{}:
if !common.StringsContains(endpoints, k) {
endpoints = append(endpoints, k)
}
endpoints = appendPricingEndpoint(endpoints, k)
}
}
if len(endpoints) > 0 {
......@@ -264,7 +333,7 @@ func updatePricing() {
continue
}
var raw map[string]interface{}
if err := json.Unmarshal([]byte(meta.Endpoints), &raw); err == nil {
if err := common.Unmarshal([]byte(meta.Endpoints), &raw); err == nil {
for k, v := range raw {
switch val := v.(type) {
case string:
......
This diff was suppressed by a .gitattributes entry.
<svg width="1600" height="900" viewBox="0 0 1600 900" fill="none" xmlns="http://www.w3.org/2000/svg">
<defs>
<linearGradient id="bg" x1="140" y1="30" x2="1450" y2="880" gradientUnits="userSpaceOnUse">
<stop offset="0" stop-color="#F3FCFF"/>
<stop offset="0.46" stop-color="#FFFFFF"/>
<stop offset="1" stop-color="#FFF6FD"/>
</linearGradient>
<linearGradient id="logoGradient" x1="430" y1="252" x2="1110" y2="616" gradientUnits="userSpaceOnUse">
<stop offset="0" stop-color="#22D3EE"/>
<stop offset="0.48" stop-color="#7C3AED"/>
<stop offset="1" stop-color="#F15BB5"/>
</linearGradient>
<linearGradient id="lineGradient" x1="180" y1="120" x2="1440" y2="780" gradientUnits="userSpaceOnUse">
<stop offset="0" stop-color="#22D3EE" stop-opacity="0.28"/>
<stop offset="0.52" stop-color="#7C3AED" stop-opacity="0.22"/>
<stop offset="1" stop-color="#F15BB5" stop-opacity="0.28"/>
</linearGradient>
<linearGradient id="softBand" x1="300" y1="190" x2="1320" y2="740" gradientUnits="userSpaceOnUse">
<stop offset="0" stop-color="#22D3EE" stop-opacity="0.14"/>
<stop offset="0.5" stop-color="#7C3AED" stop-opacity="0.1"/>
<stop offset="1" stop-color="#F15BB5" stop-opacity="0.14"/>
</linearGradient>
<filter id="softShadow" x="-20%" y="-20%" width="140%" height="140%" color-interpolation-filters="sRGB">
<feDropShadow dx="0" dy="24" stdDeviation="34" flood-color="#64748B" flood-opacity="0.16"/>
</filter>
<filter id="logoShadow" x="-30%" y="-30%" width="160%" height="160%" color-interpolation-filters="sRGB">
<feDropShadow dx="0" dy="18" stdDeviation="24" flood-color="#7C3AED" flood-opacity="0.18"/>
</filter>
<clipPath id="logoClip">
<circle cx="800" cy="206" r="62"/>
</clipPath>
</defs>
<rect width="1600" height="900" rx="0" fill="url(#bg)"/>
<g opacity="0.65">
<path d="M-90 654C145 538 294 690 495 594C702 495 678 278 908 252C1129 227 1214 421 1697 277" stroke="url(#lineGradient)" stroke-width="2.4"/>
<path d="M-62 266C184 413 331 178 539 280C772 394 782 632 1008 624C1216 617 1332 424 1664 548" stroke="url(#lineGradient)" stroke-width="2"/>
<path d="M122 780C392 624 561 779 742 622C905 480 806 323 992 222C1172 124 1322 196 1512 96" stroke="url(#lineGradient)" stroke-width="1.7"/>
</g>
<g opacity="0.22">
<path d="M204 154H1396" stroke="#7C3AED" stroke-width="1"/>
<path d="M204 746H1396" stroke="#22D3EE" stroke-width="1"/>
<path d="M280 80V820" stroke="#F15BB5" stroke-width="1"/>
<path d="M1320 80V820" stroke="#22D3EE" stroke-width="1"/>
</g>
<g filter="url(#softShadow)">
<path d="M310 648C393 543 504 498 637 512C742 524 837 579 941 559C1075 533 1173 414 1291 457C1376 488 1420 580 1428 665C1284 741 1117 784 932 790C702 797 492 750 310 648Z" fill="url(#softBand)"/>
</g>
<g opacity="0.9">
<circle cx="376" cy="232" r="7" fill="#22D3EE"/>
<circle cx="1266" cy="236" r="7" fill="#F15BB5"/>
<circle cx="1328" cy="642" r="6" fill="#7C3AED"/>
<circle cx="254" cy="606" r="5" fill="#F15BB5"/>
<circle cx="1172" cy="154" r="4" fill="#22D3EE"/>
<circle cx="506" cy="760" r="4" fill="#7C3AED"/>
</g>
<g opacity="0.82">
<path d="M1190 358L1202 386L1232 398L1202 410L1190 438L1178 410L1148 398L1178 386L1190 358Z" fill="#22D3EE"/>
<path d="M420 416L430 440L456 450L430 460L420 484L410 460L384 450L410 440L420 416Z" fill="#F15BB5"/>
<path d="M1074 718L1083 739L1105 748L1083 757L1074 778L1065 757L1043 748L1065 739L1074 718Z" fill="#7C3AED"/>
</g>
<g transform="translate(0 10)">
<g filter="url(#logoShadow)">
<circle cx="800" cy="206" r="78" fill="#FFFFFF"/>
<circle cx="800" cy="206" r="77" stroke="#E8F3FF" stroke-width="2"/>
<image href="/Users/caion/GolandProjects/new-api/web/default/public/logo.png" x="738" y="144" width="124" height="124" clip-path="url(#logoClip)" preserveAspectRatio="xMidYMid meet"/>
</g>
<text x="800" y="344" text-anchor="middle"
font-family="Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, sans-serif"
font-size="46" font-weight="760" letter-spacing="0" fill="#0F172A">NewAPI</text>
<text x="800" y="516" text-anchor="middle"
font-family="Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, sans-serif"
font-size="170" font-weight="860" letter-spacing="0" fill="url(#logoGradient)">40K</text>
<text x="800" y="616" text-anchor="middle"
font-family="Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, sans-serif"
font-size="76" font-weight="780" letter-spacing="0" fill="#101828">Stars</text>
<text x="800" y="688" text-anchor="middle"
font-family="Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, sans-serif"
font-size="34" font-weight="560" letter-spacing="0" fill="#475569">Thank you, builders</text>
</g>
<g transform="translate(104 86)">
<image href="/Users/caion/GolandProjects/new-api/web/default/public/logo.png" x="0" y="0" width="42" height="42" preserveAspectRatio="xMidYMid meet"/>
<text x="58" y="30"
font-family="Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, sans-serif"
font-size="26" font-weight="730" letter-spacing="0" fill="#162033">NewAPI</text>
</g>
<g transform="translate(1262 96)">
<rect x="0" y="0" width="234" height="54" rx="27" fill="#FFFFFF" fill-opacity="0.78" stroke="#D7E7F5"/>
<text x="117" y="35" text-anchor="middle"
font-family="Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, sans-serif"
font-size="23" font-weight="660" letter-spacing="0" fill="#334155">newapi.ai</text>
</g>
<g transform="translate(118 774)">
<path d="M0 22H246" stroke="#22D3EE" stroke-width="5" stroke-linecap="round"/>
<path d="M281 22H464" stroke="#7C3AED" stroke-width="5" stroke-linecap="round"/>
<path d="M500 22H652" stroke="#F15BB5" stroke-width="5" stroke-linecap="round"/>
</g>
</svg>
......@@ -75,10 +75,14 @@ func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayIn
return req, nil
}
oaiReq, err := service.ClaudeToOpenAIRequest(*req, info)
result, err := service.ConvertRequest(c, info, types.RelayFormatOpenAI, req)
if err != nil {
return nil, err
}
oaiReq, ok := result.Value.(*dto.GeneralOpenAIRequest)
if !ok {
return nil, fmt.Errorf("expected OpenAI chat completions request, got %T", result.Value)
}
if info.SupportStreamOptions && info.IsStream {
oaiReq.StreamOptions = &dto.StreamOptions{IncludeUsage: true}
}
......
......@@ -309,7 +309,7 @@ func DoApiRequest(a Adaptor, c *gin.Context, info *common.RelayInfo, requestBody
if err != nil {
return nil, fmt.Errorf("get request url failed: %w", err)
}
logger.LogDebug(c, "fullRequestURL: %s", fullRequestURL)
logger.LogDebug(c, "fullRequestURL: %s", common.SanitizeURLForLog(fullRequestURL))
req, err := http.NewRequest(c.Request.Method, fullRequestURL, requestBody)
if err != nil {
return nil, fmt.Errorf("new request failed: %w", err)
......@@ -339,7 +339,7 @@ func DoFormRequest(a Adaptor, c *gin.Context, info *common.RelayInfo, requestBod
if err != nil {
return nil, fmt.Errorf("get request url failed: %w", err)
}
logger.LogDebug(c, "fullRequestURL: %s", fullRequestURL)
logger.LogDebug(c, "fullRequestURL: %s", common.SanitizeURLForLog(fullRequestURL))
req, err := http.NewRequest(c.Request.Method, fullRequestURL, requestBody)
if err != nil {
return nil, fmt.Errorf("new request failed: %w", err)
......@@ -388,7 +388,7 @@ func DoWssRequest(a Adaptor, c *gin.Context, info *common.RelayInfo, requestBody
targetHeader.Set("Content-Type", c.Request.Header.Get("Content-Type"))
targetConn, _, err := websocket.DefaultDialer.Dial(fullRequestURL, targetHeader)
if err != nil {
return nil, fmt.Errorf("dial failed to %s: %w", fullRequestURL, err)
return nil, fmt.Errorf("dial failed to %s: %w", common.SanitizeURLForLog(fullRequestURL), err)
}
// send request body
//all, err := io.ReadAll(requestBody)
......
......@@ -123,10 +123,14 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn
}
// 原有的Claude模型处理逻辑
claudeReq, err := claude.RequestOpenAI2ClaudeMessage(c, *request)
result, err := service.ConvertRequest(c, info, types.RelayFormatClaude, request)
if err != nil {
return nil, errors.Wrap(err, "failed to convert openai request to claude request")
}
claudeReq, ok := result.Value.(*dto.ClaudeRequest)
if !ok {
return nil, fmt.Errorf("expected Anthropic Messages request, got %T", result.Value)
}
info.UpstreamModelName = claudeReq.Model
return claudeReq, err
}
......
......@@ -10,6 +10,7 @@ import (
"github.com/QuantumNous/new-api/dto"
"github.com/QuantumNous/new-api/relay/channel"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/service/relayconvert"
"github.com/QuantumNous/new-api/setting/model_setting"
"github.com/QuantumNous/new-api/types"
......@@ -95,7 +96,11 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn
if request == nil {
return nil, errors.New("request is nil")
}
return RequestOpenAI2ClaudeMessage(c, *request)
result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatClaude, request)
if err != nil {
return nil, err
}
return result.Value, nil
}
func (a *Adaptor) ConvertRerankRequest(c *gin.Context, relayMode int, request dto.RerankRequest) (any, error) {
......
......@@ -5,7 +5,7 @@ import (
"testing"
"github.com/QuantumNous/new-api/dto"
"github.com/QuantumNous/new-api/service"
"github.com/QuantumNous/new-api/service/relayconvert"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
......@@ -41,7 +41,7 @@ func TestResponseOpenAI2ClaudeToolUseInputIsObject(t *testing.T) {
},
},
})
resp := service.ResponseOpenAI2Claude(&dto.OpenAITextResponse{
resp := relayconvert.ResponseOpenAI2Claude(&dto.OpenAITextResponse{
Id: "chatcmpl_1",
Model: "gpt-test",
Choices: []dto.OpenAITextResponseChoice{
......@@ -322,7 +322,7 @@ func TestBuildOpenAIStyleUsageFromClaudeUsageDefaultsAggregateCacheCreationTo5m(
require.Equal(t, 0, openAIUsage.ClaudeCacheCreation1hTokens)
}
func TestRequestOpenAI2ClaudeMessage_ClaudeOpus48HighUsesAdaptiveThinking(t *testing.T) {
func TestOpenAIChatRequestToClaudeMessages_ClaudeOpus48HighUsesAdaptiveThinking(t *testing.T) {
request := dto.GeneralOpenAIRequest{
Model: "claude-opus-4-8-high",
Temperature: commonPointer(0.7),
......@@ -336,7 +336,7 @@ func TestRequestOpenAI2ClaudeMessage_ClaudeOpus48HighUsesAdaptiveThinking(t *tes
},
}
claudeRequest, err := RequestOpenAI2ClaudeMessage(nil, request)
claudeRequest, err := relayconvert.OpenAIChatRequestToClaudeMessages(nil, request)
require.NoError(t, err)
require.Equal(t, "claude-opus-4-8", claudeRequest.Model)
require.NotNil(t, claudeRequest.Thinking)
......@@ -348,7 +348,7 @@ func TestRequestOpenAI2ClaudeMessage_ClaudeOpus48HighUsesAdaptiveThinking(t *tes
require.Nil(t, claudeRequest.TopK)
}
func TestRequestOpenAI2ClaudeMessage_ClaudeOpus48ThinkingUsesAdaptiveHighEffort(t *testing.T) {
func TestOpenAIChatRequestToClaudeMessages_ClaudeOpus48ThinkingUsesAdaptiveHighEffort(t *testing.T) {
request := dto.GeneralOpenAIRequest{
Model: "claude-opus-4-8-thinking",
Temperature: commonPointer(0.7),
......@@ -362,7 +362,7 @@ func TestRequestOpenAI2ClaudeMessage_ClaudeOpus48ThinkingUsesAdaptiveHighEffort(
},
}
claudeRequest, err := RequestOpenAI2ClaudeMessage(nil, request)
claudeRequest, err := relayconvert.OpenAIChatRequestToClaudeMessages(nil, request)
require.NoError(t, err)
require.Equal(t, "claude-opus-4-8", claudeRequest.Model)
require.NotNil(t, claudeRequest.Thinking)
......
......@@ -9,7 +9,6 @@ import (
"github.com/QuantumNous/new-api/dto"
"github.com/QuantumNous/new-api/relay/channel"
"github.com/QuantumNous/new-api/relay/channel/openai"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/relay/constant"
"github.com/QuantumNous/new-api/service/relayconvert"
......@@ -45,12 +44,15 @@ func (a *Adaptor) ConvertGeminiRequest(c *gin.Context, info *relaycommon.RelayIn
}
func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayInfo, req *dto.ClaudeRequest) (any, error) {
adaptor := openai.Adaptor{}
oaiReq, err := adaptor.ConvertClaudeRequest(c, info, req)
result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatGemini, req)
if err != nil {
return nil, err
}
return a.ConvertOpenAIRequest(c, info, oaiReq.(*dto.GeneralOpenAIRequest))
geminiRequest, ok := result.Value.(*dto.GeminiChatRequest)
if !ok {
return nil, fmt.Errorf("expected Gemini generateContent request, got %T", result.Value)
}
return geminiRequest, nil
}
func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.AudioRequest) (io.Reader, error) {
......@@ -181,13 +183,11 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn
if request == nil {
return nil, errors.New("request is nil")
}
geminiRequest, err := CovertOpenAI2Gemini(c, *request, info)
result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatGemini, request)
if err != nil {
return nil, err
}
return geminiRequest, nil
return result.Value, nil
}
func (a *Adaptor) ConvertRerankRequest(c *gin.Context, relayMode int, request dto.RerankRequest) (any, error) {
......@@ -239,17 +239,15 @@ func (a *Adaptor) ConvertEmbeddingRequest(c *gin.Context, info *relaycommon.Rela
}
func (a *Adaptor) ConvertOpenAIResponsesRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.OpenAIResponsesRequest) (any, error) {
request, err := preprocessGeminiOpenAIResponsesRequest(request)
result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatGemini, &request)
if err != nil {
return nil, err
}
chatRequest, err := relayconvert.ResponsesRequestToChatCompletionsRequest(&request)
if err != nil {
return nil, err
geminiRequest, ok := result.Value.(*dto.GeminiChatRequest)
if !ok {
return nil, fmt.Errorf("expected Gemini generateContent request, got %T", result.Value)
}
return a.ConvertOpenAIRequest(c, info, chatRequest)
return geminiRequest, nil
}
func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (any, error) {
......
......@@ -39,8 +39,8 @@ func GeminiTextGenerationHandler(c *gin.Context, info *relaycommon.RelayInfo, re
common.SetContextKey(c, constant.ContextKeyAdminRejectReason, fmt.Sprintf("gemini_block_reason=%s", *geminiResponse.PromptFeedback.BlockReason))
}
// 计算使用量(基于 UsageMetadata
usage := buildUsageFromGeminiMetadata(geminiResponse.UsageMetadata, info.GetEstimatePromptTokens())
// 计算使用量(优先上游 UsageMetadata,缺失时本地估算并保留 Gemini 计费语义
usage := buildUsageFromGeminiResponse(c, info, &geminiResponse)
service.IOCopyBytesGracefully(c, resp, responseBody)
......
......@@ -331,3 +331,186 @@ func TestGeminiTextGenerationHandlerUsesEstimatedPromptTokensWhenUsagePromptMiss
require.Equal(t, 100, usage.CompletionTokens)
require.Equal(t, 110, usage.TotalTokens)
}
func TestGeminiChatHandlerMissingUsageMetadataBuildsEstimatedBillingUsage(t *testing.T) {
t.Parallel()
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
info := &relaycommon.RelayInfo{
RelayFormat: types.RelayFormatGemini,
OriginModelName: "gemini-3-flash-preview",
ChannelMeta: &relaycommon.ChannelMeta{
UpstreamModelName: "gemini-3-flash-preview",
},
}
info.SetEstimatePromptTokens(20)
body := []byte(`{"candidates":[{"content":{"role":"model","parts":[{"text":"ok"}]}}]}`)
resp := &http.Response{
Body: io.NopCloser(bytes.NewReader(body)),
}
usage, newAPIError := GeminiChatHandler(c, info, resp)
require.Nil(t, newAPIError)
require.NotNil(t, usage)
require.Equal(t, 20, usage.PromptTokens)
require.NotNil(t, usage.BillingUsage)
require.True(t, usage.BillingUsage.Estimated)
require.Equal(t, dto.BillingUsageSourceGeminiChat, usage.BillingUsage.Source)
require.Equal(t, dto.BillingUsageSemanticGemini, usage.BillingUsage.Semantic)
require.NotNil(t, usage.BillingUsage.GeminiUsageMetadata)
require.Equal(t, usage.PromptTokens, usage.BillingUsage.GeminiUsageMetadata.PromptTokenCount)
require.Equal(t, usage.CompletionTokens, usage.BillingUsage.GeminiUsageMetadata.CandidatesTokenCount)
require.True(t, common.GetContextKeyBool(c, constant.ContextKeyLocalCountTokens))
}
func TestGeminiStreamHandlerPromptOnlyUsageMetadataEstimatesCompletionTokens(t *testing.T) {
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
oldStreamingTimeout := constant.StreamingTimeout
constant.StreamingTimeout = 300
t.Cleanup(func() {
constant.StreamingTimeout = oldStreamingTimeout
})
info := &relaycommon.RelayInfo{
OriginModelName: "gemini-3-flash-preview",
ChannelMeta: &relaycommon.ChannelMeta{
UpstreamModelName: "gemini-3-flash-preview",
},
}
info.SetEstimatePromptTokens(20)
// Simulates a client aborting the stream before the final chunk: text was
// streamed but the last observed usageMetadata only carries prompt tokens.
chunk := dto.GeminiChatResponse{
Candidates: []dto.GeminiChatCandidate{
{
Content: dto.GeminiChatContent{
Role: "model",
Parts: []dto.GeminiPart{
{Text: "partial streamed answer before disconnect"},
},
},
},
},
UsageMetadata: dto.GeminiUsageMetadata{
PromptTokenCount: 151,
TotalTokenCount: 151,
},
}
chunkData, err := common.Marshal(chunk)
require.NoError(t, err)
streamBody := []byte("data: " + string(chunkData) + "\n" + "data: [DONE]\n")
resp := &http.Response{
Body: io.NopCloser(bytes.NewReader(streamBody)),
}
usage, newAPIError := geminiStreamHandler(c, info, resp, func(_ string, _ *dto.GeminiChatResponse) bool {
return true
})
require.Nil(t, newAPIError)
require.NotNil(t, usage)
require.Equal(t, 151, usage.PromptTokens)
require.Greater(t, usage.CompletionTokens, 0)
require.Equal(t, usage.PromptTokens+usage.CompletionTokens, usage.TotalTokens)
require.NotNil(t, usage.BillingUsage)
require.True(t, usage.BillingUsage.Estimated)
require.NotNil(t, usage.BillingUsage.GeminiUsageMetadata)
require.Equal(t, usage.CompletionTokens, usage.BillingUsage.GeminiUsageMetadata.CandidatesTokenCount)
}
func TestGeminiChatHandlerPromptOnlyUsageMetadataEstimatesCompletionTokens(t *testing.T) {
t.Parallel()
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
info := &relaycommon.RelayInfo{
RelayFormat: types.RelayFormatGemini,
OriginModelName: "gemini-3-flash-preview",
ChannelMeta: &relaycommon.ChannelMeta{
UpstreamModelName: "gemini-3-flash-preview",
},
}
payload := dto.GeminiChatResponse{
Candidates: []dto.GeminiChatCandidate{
{
Content: dto.GeminiChatContent{
Role: "model",
Parts: []dto.GeminiPart{
{Text: "answer text without candidate token count"},
},
},
},
},
UsageMetadata: dto.GeminiUsageMetadata{
PromptTokenCount: 151,
TotalTokenCount: 151,
},
}
body, err := common.Marshal(payload)
require.NoError(t, err)
resp := &http.Response{
Body: io.NopCloser(bytes.NewReader(body)),
}
usage, newAPIError := GeminiChatHandler(c, info, resp)
require.Nil(t, newAPIError)
require.NotNil(t, usage)
require.Equal(t, 151, usage.PromptTokens)
require.Greater(t, usage.CompletionTokens, 0)
require.Equal(t, usage.PromptTokens+usage.CompletionTokens, usage.TotalTokens)
require.NotNil(t, usage.BillingUsage)
require.True(t, usage.BillingUsage.Estimated)
}
func TestGeminiStreamHandlerEmptyUsageMetadataBuildsEstimatedBillingUsage(t *testing.T) {
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
oldStreamingTimeout := constant.StreamingTimeout
constant.StreamingTimeout = 300
t.Cleanup(func() {
constant.StreamingTimeout = oldStreamingTimeout
})
info := &relaycommon.RelayInfo{
OriginModelName: "gemini-3-flash-preview",
ChannelMeta: &relaycommon.ChannelMeta{
UpstreamModelName: "gemini-3-flash-preview",
},
}
info.SetEstimatePromptTokens(20)
streamBody := []byte("data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"partial\"}]}}],\"usageMetadata\":{}}\n" + "data: [DONE]\n")
resp := &http.Response{
Body: io.NopCloser(bytes.NewReader(streamBody)),
}
usage, newAPIError := geminiStreamHandler(c, info, resp, func(_ string, _ *dto.GeminiChatResponse) bool {
return true
})
require.Nil(t, newAPIError)
require.NotNil(t, usage)
require.Equal(t, 20, usage.PromptTokens)
require.NotNil(t, usage.BillingUsage)
require.True(t, usage.BillingUsage.Estimated)
require.Equal(t, dto.BillingUsageSourceGeminiChat, usage.BillingUsage.Source)
require.NotNil(t, usage.BillingUsage.GeminiUsageMetadata)
require.Equal(t, usage.PromptTokens, usage.BillingUsage.GeminiUsageMetadata.PromptTokenCount)
require.Equal(t, usage.CompletionTokens, usage.BillingUsage.GeminiUsageMetadata.CandidatesTokenCount)
require.True(t, common.GetContextKeyBool(c, constant.ContextKeyLocalCountTokens))
}
......@@ -32,7 +32,7 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
if len(geminiResponse.Candidates) == 0 {
usage := buildUsageFromGeminiMetadata(geminiResponse.UsageMetadata, info.GetEstimatePromptTokens())
usage := buildUsageFromGeminiResponse(c, info, &geminiResponse)
if geminiResponse.PromptFeedback != nil && geminiResponse.PromptFeedback.BlockReason != nil {
common.SetContextKey(c, constant.ContextKeyAdminRejectReason, fmt.Sprintf("gemini_block_reason=%s", *geminiResponse.PromptFeedback.BlockReason))
return &usage, types.NewOpenAIError(
......@@ -51,13 +51,21 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h
chatResp := responseGeminiChat2OpenAI(c, &geminiResponse)
chatResp.Model = info.UpstreamModelName
usage := buildUsageFromGeminiMetadata(geminiResponse.UsageMetadata, info.GetEstimatePromptTokens())
if responseID := helper.GetResponseID(c); responseID != "" {
chatResp.Id = responseID
}
usage := buildUsageFromGeminiResponse(c, info, &geminiResponse)
chatResp.Usage = usage
responsesResp, responsesUsage, err := service.ChatCompletionsResponseToResponsesResponse(chatResp, helper.GetResponseID(c))
convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatOpenAIResponses, chatResp)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
responsesResp, ok := convertResult.Value.(*dto.OpenAIResponsesResponse)
if !ok {
return nil, types.NewOpenAIError(fmt.Errorf("expected OpenAI responses response, got %T", convertResult.Value), types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
responsesUsage := convertResult.Usage
if responsesUsage == nil || responsesUsage.TotalTokens == 0 {
responsesResp.Usage = relayconvert.UsageFromChatUsage(&usage)
}
......@@ -73,8 +81,14 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h
func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) {
responseID := helper.GetResponseID(c)
created := common.GetTimestamp()
state := relayconvert.NewChatToResponsesStreamState(responseID, info.UpstreamModelName)
state.Created = created
state, err := relayconvert.NewResponseStreamState(types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses, relayconvert.ResponseStreamOptions{
ID: responseID,
Model: info.UpstreamModelName,
Created: created,
})
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
finishReason := constant.FinishReasonStop
toolCallIndexByChoice := make(map[int]map[string]int)
nextToolCallIndexByChoice := make(map[int]int)
......@@ -90,12 +104,17 @@ func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, r
return true
}
sendChunk := func(chunk *dto.ChatCompletionsStreamResponse) bool {
events, err := relayconvert.ChatCompletionsStreamChunkToResponsesEvents(chunk, state)
results, err := relayconvert.ConvertStreamResponseChunk(c, info, state, chunk)
if err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
for _, event := range events {
for _, result := range results {
event, ok := result.Value.(relayconvert.ChatToResponsesStreamEvent)
if !ok {
streamErr = types.NewOpenAIError(fmt.Errorf("expected OAI responses stream event, got %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
if !sendEvent(event) {
return false
}
......@@ -103,7 +122,7 @@ func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, r
return true
}
usage, err := geminiStreamHandler(c, info, resp, func(data string, geminiResponse *dto.GeminiChatResponse) bool {
usage, streamAPIError := geminiStreamHandler(c, info, resp, func(data string, geminiResponse *dto.GeminiChatResponse) bool {
response, isStop := streamResponseGeminiChat2OpenAI(geminiResponse)
response.Id = responseID
response.Created = created
......@@ -143,17 +162,25 @@ func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, r
}
return true
})
if err != nil {
return usage, err
if streamAPIError != nil {
return usage, streamAPIError
}
if streamErr != nil {
return nil, streamErr
}
if usage != nil {
state.Usage = relayconvert.UsageFromChatUsage(usage)
state.SetUsage(usage)
}
for _, event := range relayconvert.FinalizeChatCompletionsStreamToResponses(state) {
finalResults, err := relayconvert.FinalizeStreamResponse(c, info, state)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
for _, result := range finalResults {
event, ok := result.Value.(relayconvert.ChatToResponsesStreamEvent)
if !ok {
return nil, types.NewOpenAIError(fmt.Errorf("expected OAI responses stream event, got %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
if !sendEvent(event) {
return nil, streamErr
}
......
......@@ -42,11 +42,14 @@ type Adaptor struct {
}
func (a *Adaptor) ConvertGeminiRequest(c *gin.Context, info *relaycommon.RelayInfo, request *dto.GeminiChatRequest) (any, error) {
// 使用 service.GeminiToOpenAIRequest 转换请求格式
openaiRequest, err := service.GeminiToOpenAIRequest(request, info)
result, err := service.ConvertRequest(c, info, types.RelayFormatOpenAI, request)
if err != nil {
return nil, err
}
openaiRequest, ok := result.Value.(*dto.GeneralOpenAIRequest)
if !ok {
return nil, fmt.Errorf("expected OpenAI chat completions request, got %T", result.Value)
}
return a.ConvertOpenAIRequest(c, info, openaiRequest)
}
......@@ -61,10 +64,14 @@ func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayIn
// println(fmt.Sprintf("failed to save request body to file: %v", err))
// }
//}
aiRequest, err := service.ClaudeToOpenAIRequest(*request, info)
result, err := service.ConvertRequest(c, info, types.RelayFormatOpenAI, request)
if err != nil {
return nil, err
}
aiRequest, ok := result.Value.(*dto.GeneralOpenAIRequest)
if !ok {
return nil, fmt.Errorf("expected OpenAI chat completions request, got %T", result.Value)
}
//if common.DebugEnabled {
// println(fmt.Sprintf("convert claude to openai request result: %s", common.GetJsonString(aiRequest)))
// // Save request body to file for debugging
......
......@@ -41,11 +41,18 @@ func OaiResponsesToChatHandler(c *gin.Context, info *relaycommon.RelayInfo, resp
return nil, types.WithOpenAIError(*oaiError, resp.StatusCode)
}
chatId := helper.GetResponseID(c)
chatResp, usage, err := service.ResponsesResponseToChatCompletionsResponse(&responsesResp, chatId)
chatResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatOpenAI, &responsesResp)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
chatResp, ok := chatResult.Value.(*dto.OpenAITextResponse)
if !ok {
return nil, types.NewOpenAIError(fmt.Errorf("expected OpenAI chat response, got %T", chatResult.Value), types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
if chatID := helper.GetResponseID(c); chatID != "" {
chatResp.Id = chatID
}
usage := chatResult.Usage
if usage == nil || usage.TotalTokens == 0 {
text := service.ExtractOutputTextFromResponses(&responsesResp)
......@@ -53,17 +60,15 @@ func OaiResponsesToChatHandler(c *gin.Context, info *relaycommon.RelayInfo, resp
chatResp.Usage = *usage
}
var responseBody []byte
switch info.RelayFormat {
case types.RelayFormatClaude:
claudeResp := service.ResponseOpenAI2Claude(chatResp, info)
responseBody, err = common.Marshal(claudeResp)
case types.RelayFormatGemini:
geminiResp := service.ResponseOpenAI2Gemini(chatResp, info)
responseBody, err = common.Marshal(geminiResp)
default:
responseBody, err = common.Marshal(chatResp)
responseValue := any(chatResp)
if info.RelayFormat != types.RelayFormatOpenAI {
targetResult, err := relayconvert.ConvertResponse(c, info, info.RelayFormat, chatResp)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
responseValue = targetResult.Value
}
responseBody, err := common.Marshal(responseValue)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError)
}
......@@ -145,28 +150,33 @@ func OaiResponsesToChatBufferedStreamHandler(c *gin.Context, info *relaycommon.R
}
accumulator.SupplementResponseOutput(finalResponse)
chatId := helper.GetResponseID(c)
chatResp, usage, err := service.ResponsesResponseToChatCompletionsResponse(finalResponse, chatId)
chatResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatOpenAI, finalResponse)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
chatResp, ok := chatResult.Value.(*dto.OpenAITextResponse)
if !ok {
return nil, types.NewOpenAIError(fmt.Errorf("expected OpenAI chat response, got %T", chatResult.Value), types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
if chatID := helper.GetResponseID(c); chatID != "" {
chatResp.Id = chatID
}
usage := chatResult.Usage
if usage == nil || usage.TotalTokens == 0 {
text := service.ExtractOutputTextFromResponses(finalResponse)
usage = service.ResponseText2Usage(c, text, info.UpstreamModelName, info.GetEstimatePromptTokens())
chatResp.Usage = *usage
}
var responseBody []byte
switch info.RelayFormat {
case types.RelayFormatClaude:
claudeResp := service.ResponseOpenAI2Claude(chatResp, info)
responseBody, err = common.Marshal(claudeResp)
case types.RelayFormatGemini:
geminiResp := service.ResponseOpenAI2Gemini(chatResp, info)
responseBody, err = common.Marshal(geminiResp)
default:
responseBody, err = common.Marshal(chatResp)
responseValue := any(chatResp)
if info.RelayFormat != types.RelayFormatOpenAI {
targetResult, err := relayconvert.ConvertResponse(c, info, info.RelayFormat, chatResp)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
responseValue = targetResult.Value
}
responseBody, err := common.Marshal(responseValue)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError)
}
......@@ -184,37 +194,77 @@ func OaiResponsesToChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo
responseId := helper.GetResponseID(c)
createAt := time.Now().Unix()
state := relayconvert.NewResponsesToChatStreamState(info.UpstreamModelName, false)
state.ID = responseId
state.Created = createAt
state, err := relayconvert.NewResponseStreamState(types.RelayFormatOpenAIResponses, info.RelayFormat, relayconvert.ResponseStreamOptions{
ID: responseId,
Model: info.UpstreamModelName,
Created: createAt,
})
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
streamErr := (*types.NewAPIError)(nil)
if info.RelayFormat == types.RelayFormatClaude && info.ClaudeConvertInfo == nil {
info.ClaudeConvertInfo = &relaycommon.ClaudeConvertInfo{LastMessagesType: relaycommon.LastMessageTypeNone}
}
sendChatChunk := func(chunk dto.ChatCompletionsStreamResponse) bool {
if len(chunk.Choices) == 0 && chunk.Usage == nil {
sendGeminiResponse := func(geminiResponse *dto.GeminiChatResponse) bool {
if geminiResponse == nil {
return true
}
if info.RelayFormat == types.RelayFormatOpenAI {
if err := helper.ObjectData(c, &chunk); err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
return true
}
chunkData, err := common.Marshal(&chunk)
geminiResponseStr, err := common.Marshal(geminiResponse)
if err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError)
return false
}
if err := HandleStreamFormat(c, info, string(chunkData), false, false); err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
c.Render(-1, common.CustomEvent{Data: "data: " + string(geminiResponseStr)})
_ = helper.FlushWriter(c)
return true
}
sendStreamResult := func(result relayconvert.ResponseResult) bool {
switch value := result.Value.(type) {
case dto.ChatCompletionsStreamResponse:
if len(value.Choices) == 0 && value.Usage == nil {
return true
}
if err := helper.ObjectData(c, &value); err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
return true
case *dto.ChatCompletionsStreamResponse:
if value == nil || (len(value.Choices) == 0 && value.Usage == nil) {
return true
}
if err := helper.ObjectData(c, value); err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
return true
case dto.ClaudeResponse:
if err := helper.ClaudeData(c, value); err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
return true
case *dto.ClaudeResponse:
if value == nil {
return true
}
if err := helper.ClaudeData(c, *value); err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
return true
case dto.GeminiChatResponse:
return sendGeminiResponse(&value)
case *dto.GeminiChatResponse:
return sendGeminiResponse(value)
default:
streamErr = types.NewOpenAIError(fmt.Errorf("unsupported converted stream response type %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
return true
}
helper.StreamScannerHandler(c, resp, info, func(data string, sr *helper.StreamResult) {
......@@ -243,14 +293,14 @@ func OaiResponsesToChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo
return
}
chunks, err := relayconvert.ResponsesStreamEventToChatChunks(&streamResp, state)
results, err := relayconvert.ConvertStreamResponseChunk(c, info, state, &streamResp)
if err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
sr.Stop(streamErr)
return
}
for _, chunk := range chunks {
if !sendChatChunk(chunk) {
for _, result := range results {
if !sendStreamResult(result) {
sr.Stop(streamErr)
return
}
......@@ -261,22 +311,26 @@ func OaiResponsesToChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo
return nil, streamErr
}
usage := state.Usage
if usage.TotalTokens == 0 {
usage := state.Usage()
if usage == nil || usage.TotalTokens == 0 {
usage = service.ResponseText2Usage(c, state.UsageText(), info.UpstreamModelName, info.GetEstimatePromptTokens())
state.Usage = usage
state.SetUsage(usage)
}
if info.RelayFormat == types.RelayFormatClaude && info.ClaudeConvertInfo != nil {
info.ClaudeConvertInfo.Usage = usage
}
for _, chunk := range relayconvert.FinalizeResponsesToChatStream(state) {
if !sendChatChunk(chunk) {
finalResults, err := relayconvert.FinalizeStreamResponse(c, info, state)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
for _, result := range finalResults {
if !sendStreamResult(result) {
return nil, streamErr
}
}
if info.RelayFormat == types.RelayFormatOpenAI && info.ShouldIncludeUsage && usage != nil {
if err := helper.ObjectData(c, helper.GenerateFinalUsageResponse(responseId, state.Created, state.Model, *usage)); err != nil {
if err := helper.ObjectData(c, helper.GenerateFinalUsageResponse(responseId, createAt, info.UpstreamModelName, *usage)); err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
}
......
package openai
import (
"fmt"
"strings"
"github.com/QuantumNous/new-api/common"
......@@ -10,6 +11,7 @@ import (
relayconstant "github.com/QuantumNous/new-api/relay/constant"
"github.com/QuantumNous/new-api/relay/helper"
"github.com/QuantumNous/new-api/service"
"github.com/QuantumNous/new-api/service/relayconvert"
"github.com/QuantumNous/new-api/types"
"github.com/samber/lo"
......@@ -41,7 +43,14 @@ func handleClaudeFormat(c *gin.Context, data string, info *relaycommon.RelayInfo
if streamResponse.Usage != nil {
info.ClaudeConvertInfo.Usage = streamResponse.Usage
}
claudeResponses := service.StreamResponseOpenAI2Claude(&streamResponse, info)
result, err := relayconvert.ConvertStreamResponse(c, info, types.RelayFormatClaude, &streamResponse)
if err != nil {
return err
}
claudeResponses, ok := result.Value.([]*dto.ClaudeResponse)
if !ok {
return fmt.Errorf("expected Claude stream responses, got %T", result.Value)
}
for _, resp := range claudeResponses {
helper.ClaudeData(c, *resp)
}
......@@ -55,7 +64,14 @@ func handleGeminiFormat(c *gin.Context, data string, info *relaycommon.RelayInfo
return err
}
geminiResponse := service.StreamResponseOpenAI2Gemini(&streamResponse, info)
result, err := relayconvert.ConvertStreamResponse(c, info, types.RelayFormatGemini, &streamResponse)
if err != nil {
return err
}
geminiResponse, ok := result.Value.(*dto.GeminiChatResponse)
if !ok {
return fmt.Errorf("expected Gemini stream response, got %T", result.Value)
}
// 如果返回 nil,表示没有实际内容,跳过发送
if geminiResponse == nil {
......@@ -165,7 +181,16 @@ func HandleFinalResponse(c *gin.Context, info *relaycommon.RelayInfo, lastStream
info.ClaudeConvertInfo.Usage = usage
claudeResponses := service.StreamResponseOpenAI2Claude(&streamResponse, info)
result, err := relayconvert.ConvertStreamResponse(c, info, types.RelayFormatClaude, &streamResponse)
if err != nil {
common.SysLog("error converting Claude stream response: " + err.Error())
return
}
claudeResponses, ok := result.Value.([]*dto.ClaudeResponse)
if !ok {
common.SysLog(fmt.Sprintf("expected Claude stream responses, got %T", result.Value))
return
}
for _, resp := range claudeResponses {
_ = helper.ClaudeData(c, *resp)
}
......@@ -183,7 +208,16 @@ func HandleFinalResponse(c *gin.Context, info *relaycommon.RelayInfo, lastStream
// 而包含最后一段文本输出的响应(倒数第二个)的 finishReason 为 null
// 暂不知是否有程序会不兼容。
geminiResponse := service.StreamResponseOpenAI2Gemini(&streamResponse, info)
result, err := relayconvert.ConvertStreamResponse(c, info, types.RelayFormatGemini, &streamResponse)
if err != nil {
common.SysLog("error converting Gemini stream response: " + err.Error())
return
}
geminiResponse, ok := result.Value.(*dto.GeminiChatResponse)
if !ok {
common.SysLog(fmt.Sprintf("expected Gemini stream response, got %T", result.Value))
return
}
// openai 流响应开头的空数据
if geminiResponse == nil {
......
......@@ -14,6 +14,7 @@ import (
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/relay/helper"
"github.com/QuantumNous/new-api/service"
"github.com/QuantumNous/new-api/service/relayconvert"
"github.com/QuantumNous/new-api/types"
"github.com/gin-gonic/gin"
......@@ -271,15 +272,21 @@ func OpenaiHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respo
break
}
case types.RelayFormatClaude:
claudeResp := service.ResponseOpenAI2Claude(&simpleResponse, info)
claudeRespStr, err := common.Marshal(claudeResp)
convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatClaude, &simpleResponse)
if err != nil {
return nil, types.NewError(err, types.ErrorCodeBadResponseBody)
}
claudeRespStr, err := common.Marshal(convertResult.Value)
if err != nil {
return nil, types.NewError(err, types.ErrorCodeBadResponseBody)
}
responseBody = claudeRespStr
case types.RelayFormatGemini:
geminiResp := service.ResponseOpenAI2Gemini(&simpleResponse, info)
geminiRespStr, err := common.Marshal(geminiResp)
convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatGemini, &simpleResponse)
if err != nil {
return nil, types.NewError(err, types.ErrorCodeBadResponseBody)
}
geminiRespStr, err := common.Marshal(convertResult.Value)
if err != nil {
return nil, types.NewError(err, types.ErrorCodeBadResponseBody)
}
......
......@@ -35,11 +35,18 @@ func OaiChatToResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp
return nil, types.WithOpenAIError(*oaiError, resp.StatusCode)
}
responseID := helper.GetResponseID(c)
responsesResp, usage, err := service.ChatCompletionsResponseToResponsesResponse(&chatResp, responseID)
if responseID := helper.GetResponseID(c); responseID != "" {
chatResp.Id = responseID
}
convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatOpenAIResponses, &chatResp)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
responsesResp, ok := convertResult.Value.(*dto.OpenAIResponsesResponse)
if !ok {
return nil, types.NewOpenAIError(fmt.Errorf("expected OpenAI responses response, got %T", convertResult.Value), types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
usage := convertResult.Usage
if usage == nil || usage.TotalTokens == 0 {
text := service.ExtractOutputTextFromResponses(responsesResp)
usage = service.ResponseText2Usage(c, text, info.UpstreamModelName, info.GetEstimatePromptTokens())
......@@ -62,7 +69,13 @@ func OaiChatToResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo
defer service.CloseResponseBodyGracefully(resp)
responseID := helper.GetResponseID(c)
state := relayconvert.NewChatToResponsesStreamState(responseID, info.UpstreamModelName)
state, err := relayconvert.NewResponseStreamState(types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses, relayconvert.ResponseStreamOptions{
ID: responseID,
Model: info.UpstreamModelName,
})
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
streamErr := (*types.NewAPIError)(nil)
sendEvent := func(event relayconvert.ChatToResponsesStreamEvent) bool {
......@@ -97,13 +110,19 @@ func OaiChatToResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo
return
}
events, err := relayconvert.ChatCompletionsStreamChunkToResponsesEvents(&chunk, state)
results, err := relayconvert.ConvertStreamResponseChunk(c, info, state, &chunk)
if err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
sr.Stop(streamErr)
return
}
for _, event := range events {
for _, result := range results {
event, ok := result.Value.(relayconvert.ChatToResponsesStreamEvent)
if !ok {
streamErr = types.NewOpenAIError(fmt.Errorf("expected OAI responses stream event, got %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError)
sr.Stop(streamErr)
return
}
if !sendEvent(event) {
sr.Stop(streamErr)
return
......@@ -115,13 +134,21 @@ func OaiChatToResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo
return nil, streamErr
}
usage := state.Usage
usage := state.Usage()
if usage == nil || usage.TotalTokens == 0 {
usage = service.ResponseText2Usage(c, state.UsageText(), info.UpstreamModelName, info.GetEstimatePromptTokens())
state.Usage = relayconvert.UsageFromChatUsage(usage)
state.SetUsage(usage)
}
for _, event := range relayconvert.FinalizeChatCompletionsStreamToResponses(state) {
finalResults, err := relayconvert.FinalizeStreamResponse(c, info, state)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
for _, result := range finalResults {
event, ok := result.Value.(relayconvert.ChatToResponsesStreamEvent)
if !ok {
return nil, types.NewOpenAIError(fmt.Errorf("expected OAI responses stream event, got %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
if !sendEvent(event) {
return nil, streamErr
}
......
package vertex
import (
"encoding/json"
"errors"
"fmt"
"io"
......@@ -16,6 +15,7 @@ import (
"github.com/QuantumNous/new-api/relay/channel/openai"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/relay/constant"
"github.com/QuantumNous/new-api/service"
"github.com/QuantumNous/new-api/setting/model_setting"
"github.com/QuantumNous/new-api/setting/reasoning"
"github.com/QuantumNous/new-api/types"
......@@ -267,7 +267,7 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn
}
if len(request.ExtraBody) > 0 {
var extra map[string]any
if err := json.Unmarshal(request.ExtraBody, &extra); err == nil {
if err := common.Unmarshal(request.ExtraBody, &extra); err == nil {
if n, ok := extra["n"].(float64); ok && n > 0 {
imgReq.N = lo.ToPtr(uint(n))
}
......@@ -289,19 +289,27 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn
return a.ConvertImageRequest(c, info, imgReq)
}
if a.RequestMode == RequestModeClaude {
claudeReq, err := claude.RequestOpenAI2ClaudeMessage(c, *request)
result, err := service.ConvertRequest(c, info, types.RelayFormatClaude, request)
if err != nil {
return nil, err
}
claudeReq, ok := result.Value.(*dto.ClaudeRequest)
if !ok {
return nil, fmt.Errorf("expected Anthropic Messages request, got %T", result.Value)
}
vertexClaudeReq := copyRequest(claudeReq, anthropicVersion)
c.Set("request_model", claudeReq.Model)
info.UpstreamModelName = claudeReq.Model
return vertexClaudeReq, nil
} else if a.RequestMode == RequestModeGemini {
geminiRequest, err := gemini.CovertOpenAI2Gemini(c, *request, info)
result, err := service.ConvertRequest(c, info, types.RelayFormatGemini, request)
if err != nil {
return nil, err
}
geminiRequest, ok := result.Value.(*dto.GeminiChatRequest)
if !ok {
return nil, fmt.Errorf("expected Gemini generateContent request, got %T", result.Value)
}
c.Set("request_model", request.Model)
return geminiRequest, nil
} else if a.RequestMode == RequestModeOpenSource {
......
package relay
import (
"fmt"
"io"
"net/http"
"strings"
......@@ -92,11 +93,14 @@ func chatCompletionsViaResponses(c *gin.Context, info *relaycommon.RelayInfo, ad
return nil, types.NewError(err, types.ErrorCodeChannelParamOverrideInvalid, types.ErrOptionWithSkipRetry())
}
responsesReq, err := service.ChatCompletionsRequestToResponsesRequest(&overriddenChatReq)
result, err := service.ConvertRequestVia(c, info, &overriddenChatReq, types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses)
if err != nil {
return nil, types.NewErrorWithStatusCode(err, types.ErrorCodeInvalidRequest, http.StatusBadRequest, types.ErrOptionWithSkipRetry())
}
info.AppendRequestConversion(types.RelayFormatOpenAIResponses)
responsesReq, ok := result.Value.(*dto.OpenAIResponsesRequest)
if !ok {
return nil, types.NewError(fmt.Errorf("expected OpenAI responses request, got %T", result.Value), types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry())
}
savedRelayMode := info.RelayMode
savedRequestURLPath := info.RequestURLPath
......
......@@ -135,10 +135,14 @@ func ClaudeHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *typ
if !model_setting.GetGlobalSettings().PassThroughRequestEnabled &&
!info.ChannelSetting.PassThroughBodyEnabled &&
service.ShouldChatCompletionsUseResponsesGlobal(info.ChannelId, info.ChannelType, info.OriginModelName) {
openAIRequest, convErr := service.ClaudeToOpenAIRequest(*request, info)
result, convErr := service.ConvertRequest(c, info, types.RelayFormatOpenAI, request)
if convErr != nil {
return types.NewError(convErr, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry())
}
openAIRequest, ok := result.Value.(*dto.GeneralOpenAIRequest)
if !ok {
return types.NewError(fmt.Errorf("expected OpenAI chat completions request, got %T", result.Value), types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry())
}
usage, newApiErr := chatCompletionsViaResponses(c, info, adaptor, openAIRequest)
if newApiErr != nil {
......
......@@ -3,6 +3,7 @@ package common
import (
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
......@@ -36,6 +37,67 @@ func GetFullRequestURL(baseURL string, requestURL string, channelType int) strin
return fullRequestURL
}
func SanitizeURLForLog(rawURL string) string {
if rawURL == "" {
return rawURL
}
parsedURL, err := url.Parse(rawURL)
if err != nil {
return rawURL
}
query := parsedURL.Query()
if len(query) == 0 {
return rawURL
}
changed := false
for key := range query {
if isSensitiveURLQueryKey(key) {
query.Set(key, "***masked***")
changed = true
}
}
if !changed {
return rawURL
}
parsedURL.RawQuery = query.Encode()
return parsedURL.String()
}
func isSensitiveURLQueryKey(key string) bool {
normalized := strings.ToLower(strings.TrimSpace(key))
switch normalized {
case "key",
"api_key",
"api-key",
"apikey",
"x-api-key",
"access_token",
"refresh_token",
"id_token",
"token",
"authorization",
"auth",
"client_secret",
"secret",
"password",
"passwd",
"signature",
"sig",
"awsaccesskeyid",
"x-amz-credential",
"x-amz-security-token",
"x-amz-signature":
return true
}
return strings.Contains(normalized, "token") ||
strings.Contains(normalized, "secret") ||
strings.Contains(normalized, "signature")
}
func GetAPIVersion(c *gin.Context) string {
query := c.Request.URL.Query()
apiVersion := query.Get("api-version")
......
......@@ -3,14 +3,59 @@ package common
import (
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/QuantumNous/new-api/constant"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSanitizeURLForLogMasksSensitiveQueryValues(t *testing.T) {
rawURL := "https://example.test/v1beta/models/gemini:streamGenerateContent?alt=sse&key=sk-secret&access_token=ya29-secret&api-version=2024-02-01"
got := SanitizeURLForLog(rawURL)
assert.NotContains(t, got, "sk-secret")
assert.NotContains(t, got, "ya29-secret")
parsedURL, err := url.Parse(got)
require.NoError(t, err)
query := parsedURL.Query()
assert.Equal(t, "***masked***", query.Get("key"))
assert.Equal(t, "***masked***", query.Get("access_token"))
assert.Equal(t, "sse", query.Get("alt"))
assert.Equal(t, "2024-02-01", query.Get("api-version"))
}
func TestSanitizeURLForLogMasksAWSAndSecretLikeQueryKeys(t *testing.T) {
rawURL := "https://example.test/path?X-Amz-Credential=credential&X-Amz-Signature=signature&session_token=session&client_secret=secret&model=gpt-test"
got := SanitizeURLForLog(rawURL)
assert.NotContains(t, got, "X-Amz-Credential=credential")
assert.NotContains(t, got, "X-Amz-Signature=signature")
assert.NotContains(t, got, "session_token=session")
assert.NotContains(t, got, "client_secret=secret")
parsedURL, err := url.Parse(got)
require.NoError(t, err)
query := parsedURL.Query()
assert.Equal(t, "***masked***", query.Get("X-Amz-Credential"))
assert.Equal(t, "***masked***", query.Get("X-Amz-Signature"))
assert.Equal(t, "***masked***", query.Get("session_token"))
assert.Equal(t, "***masked***", query.Get("client_secret"))
assert.Equal(t, "gpt-test", query.Get("model"))
}
func TestSanitizeURLForLogKeepsURLWithoutSensitiveQuery(t *testing.T) {
rawURL := "https://example.test/v1/chat/completions?api-version=2024-02-01&alt=sse"
got := SanitizeURLForLog(rawURL)
assert.Equal(t, rawURL, got)
}
func TestValidateMultipartDirectNormalizesImageField(t *testing.T) {
gin.SetMode(gin.TestMode)
body := strings.NewReader(`{"model":"wan2.7-i2v","prompt":"animate","image":" https://example.com/first.png "}`)
......
......@@ -10,10 +10,10 @@ import (
"github.com/QuantumNous/new-api/constant"
"github.com/QuantumNous/new-api/dto"
"github.com/QuantumNous/new-api/logger"
"github.com/QuantumNous/new-api/relay/channel/gemini"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/relay/helper"
"github.com/QuantumNous/new-api/service"
"github.com/QuantumNous/new-api/service/relayconvert"
"github.com/QuantumNous/new-api/setting/model_setting"
"github.com/QuantumNous/new-api/types"
......@@ -84,7 +84,7 @@ func GeminiHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *typ
}
}
if request.GenerationConfig.ThinkingConfig == nil {
gemini.ThinkingAdaptor(request, info)
relayconvert.ApplyGeminiThinkingConfig(request, info)
}
}
......
package service
import (
"strings"
"github.com/QuantumNous/new-api/dto"
)
const (
usageBillingPathLocal = "local"
usageBillingPathUpstream = "upstream"
usageBillingPathOpenAI = "billing-usage-openai"
usageBillingPathOpenAIEstimated = "billing-usage-openai-estimated"
usageBillingPathAnthropic = "billing-usage-anthropic"
usageBillingPathAnthropicEstimated = "billing-usage-anthropic-estimated"
usageBillingPathGemini = "billing-usage-gemini"
usageBillingPathGeminiEstimated = "billing-usage-gemini-estimated"
)
func effectiveBillingUsage(usage *dto.Usage) *dto.Usage {
if billingUsage, ok := usageFromBillingUsage(usage); ok {
return billingUsage
}
return usage
}
func usageBillingPathForLog(isLocalCountTokens bool, usage *dto.Usage) string {
if isLocalCountTokens {
return usageBillingPathLocal
}
if usage == nil || usage.BillingUsage == nil {
return usageBillingPathUpstream
}
source := strings.TrimSpace(usage.BillingUsage.Source)
semantic := strings.TrimSpace(usage.BillingUsage.Semantic)
if strings.EqualFold(source, dto.BillingUsageSourceOAIChat) ||
strings.EqualFold(source, dto.BillingUsageSourceOAIResponses) ||
strings.EqualFold(semantic, dto.BillingUsageSemanticOpenAI) {
if usage.BillingUsage.Estimated {
return usageBillingPathOpenAIEstimated
}
return usageBillingPathOpenAI
}
if strings.EqualFold(source, dto.BillingUsageSourceClaudeMessages) ||
strings.EqualFold(semantic, dto.BillingUsageSemanticAnthropic) {
if usage.BillingUsage.Estimated {
return usageBillingPathAnthropicEstimated
}
return usageBillingPathAnthropic
}
if strings.EqualFold(source, dto.BillingUsageSourceGeminiChat) ||
strings.EqualFold(semantic, dto.BillingUsageSemanticGemini) {
if usage.BillingUsage.Estimated {
return usageBillingPathGeminiEstimated
}
return usageBillingPathGemini
}
return usageBillingPathUpstream
}
func appendUsageBillingPathForLog(other map[string]interface{}, isLocalCountTokens bool, usage *dto.Usage) {
if other == nil {
return
}
adminInfo, ok := other["admin_info"].(map[string]interface{})
if !ok || adminInfo == nil {
adminInfo = make(map[string]interface{})
other["admin_info"] = adminInfo
}
adminInfo["usage_billing_path"] = usageBillingPathForLog(isLocalCountTokens, usage)
}
func usageFromBillingUsage(usage *dto.Usage) (*dto.Usage, bool) {
if usage == nil || usage.BillingUsage == nil {
return nil, false
}
billingUsage := usage.BillingUsage
source := strings.TrimSpace(billingUsage.Source)
semantic := strings.TrimSpace(billingUsage.Semantic)
if billingUsage.OpenAIUsage != nil &&
(strings.EqualFold(source, dto.BillingUsageSourceOAIChat) ||
strings.EqualFold(source, dto.BillingUsageSourceOAIResponses) ||
strings.EqualFold(semantic, dto.BillingUsageSemanticOpenAI)) {
return usageFromOpenAIBillingUsage(billingUsage), true
}
if billingUsage.ClaudeUsage != nil &&
(strings.EqualFold(source, dto.BillingUsageSourceClaudeMessages) ||
strings.EqualFold(semantic, dto.BillingUsageSemanticAnthropic)) {
return usageFromClaudeBillingUsage(billingUsage), true
}
if billingUsage.GeminiUsageMetadata != nil &&
(strings.EqualFold(source, dto.BillingUsageSourceGeminiChat) ||
strings.EqualFold(semantic, dto.BillingUsageSemanticGemini)) {
return usageFromGeminiBillingUsage(billingUsage), true
}
return nil, false
}
func usageFromOpenAIBillingUsage(billingUsage *dto.BillingUsage) *dto.Usage {
usage := *billingUsage.OpenAIUsage
if usage.PromptTokens == 0 && usage.InputTokens > 0 {
usage.PromptTokens = usage.InputTokens
}
if usage.CompletionTokens == 0 && usage.OutputTokens > 0 {
usage.CompletionTokens = usage.OutputTokens
}
if usage.InputTokens == 0 && usage.PromptTokens > 0 {
usage.InputTokens = usage.PromptTokens
}
if usage.OutputTokens == 0 && usage.CompletionTokens > 0 {
usage.OutputTokens = usage.CompletionTokens
}
if usage.TotalTokens == 0 {
usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens
}
usage.UsageSemantic = dto.BillingUsageSemanticOpenAI
usage.UsageSource = billingUsage.Source
usage.BillingUsage = dto.CloneBillingUsage(billingUsage)
return &usage
}
func usageFromClaudeBillingUsage(billingUsage *dto.BillingUsage) *dto.Usage {
claudeUsage := billingUsage.ClaudeUsage
cacheCreation5m := claudeUsage.GetCacheCreation5mTokens()
if cacheCreation5m == 0 {
cacheCreation5m = claudeUsage.ClaudeCacheCreation5mTokens
}
cacheCreation1h := claudeUsage.GetCacheCreation1hTokens()
if cacheCreation1h == 0 {
cacheCreation1h = claudeUsage.ClaudeCacheCreation1hTokens
}
usage := &dto.Usage{
PromptTokens: claudeUsage.InputTokens,
CompletionTokens: claudeUsage.OutputTokens,
TotalTokens: claudeUsage.InputTokens + claudeUsage.OutputTokens,
InputTokens: claudeUsage.InputTokens + claudeUsage.CacheReadInputTokens + claudeUsage.CacheCreationInputTokens,
OutputTokens: claudeUsage.OutputTokens,
UsageSemantic: dto.BillingUsageSemanticAnthropic,
UsageSource: dto.BillingUsageSourceClaudeMessages,
BillingUsage: dto.CloneBillingUsage(billingUsage),
ClaudeCacheCreation5mTokens: cacheCreation5m,
ClaudeCacheCreation1hTokens: cacheCreation1h,
}
usage.PromptTokensDetails.CachedTokens = claudeUsage.CacheReadInputTokens
usage.PromptTokensDetails.CachedCreationTokens = claudeUsage.CacheCreationInputTokens
return usage
}
func usageFromGeminiBillingUsage(billingUsage *dto.BillingUsage) *dto.Usage {
metadata := *billingUsage.GeminiUsageMetadata
promptTokens := metadata.PromptTokenCount + metadata.ToolUsePromptTokenCount
usage := &dto.Usage{
PromptTokens: promptTokens,
CompletionTokens: metadata.CandidatesTokenCount + metadata.ThoughtsTokenCount,
TotalTokens: metadata.TotalTokenCount,
UsageSemantic: dto.BillingUsageSemanticGemini,
UsageSource: dto.BillingUsageSourceGeminiChat,
BillingUsage: dto.CloneBillingUsage(billingUsage),
}
usage.CompletionTokenDetails.ReasoningTokens = metadata.ThoughtsTokenCount
usage.PromptTokensDetails.CachedTokens = metadata.CachedContentTokenCount
for _, detail := range metadata.PromptTokensDetails {
addGeminiInputTokenDetail(&usage.PromptTokensDetails, detail)
}
for _, detail := range metadata.ToolUsePromptTokensDetails {
addGeminiInputTokenDetail(&usage.PromptTokensDetails, detail)
}
for _, detail := range metadata.CandidatesTokensDetails {
switch detail.Modality {
case "IMAGE":
usage.CompletionTokenDetails.ImageTokens += detail.TokenCount
case "AUDIO":
usage.CompletionTokenDetails.AudioTokens += detail.TokenCount
case "TEXT":
usage.CompletionTokenDetails.TextTokens += detail.TokenCount
}
}
if usage.TotalTokens == 0 {
usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens
} else if usage.CompletionTokens <= 0 {
usage.CompletionTokens = usage.TotalTokens - usage.PromptTokens
}
if usage.PromptTokens > 0 && usage.PromptTokensDetails.TextTokens == 0 && usage.PromptTokensDetails.AudioTokens == 0 {
usage.PromptTokensDetails.TextTokens = usage.PromptTokens
}
return usage
}
func addGeminiInputTokenDetail(details *dto.InputTokenDetails, detail dto.GeminiPromptTokensDetails) {
switch detail.Modality {
case "AUDIO":
details.AudioTokens += detail.TokenCount
case "IMAGE":
details.ImageTokens += detail.TokenCount
case "TEXT":
details.TextTokens += detail.TokenCount
}
}
package service
import (
"testing"
"github.com/QuantumNous/new-api/dto"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestResponseConverterFacades(t *testing.T) {
cache5m, cache1h := NormalizeCacheCreationSplit(10, 3, 2)
assert.Equal(t, 8, cache5m)
assert.Equal(t, 2, cache1h)
chatResp := &dto.OpenAITextResponse{
Id: "chatcmpl_1",
Model: "gpt-test",
Choices: []dto.OpenAITextResponseChoice{
{
Message: dto.Message{
Role: "assistant",
Content: "hello",
},
FinishReason: "stop",
},
},
}
claudeResp := ResponseOpenAI2Claude(chatResp, &relaycommon.RelayInfo{})
require.NotNil(t, claudeResp)
assert.Equal(t, "message", claudeResp.Type)
geminiResp := ResponseOpenAI2Gemini(chatResp, &relaycommon.RelayInfo{})
require.NotNil(t, geminiResp)
require.Len(t, geminiResp.Candidates, 1)
}
func TestStreamResponseConverterFacades(t *testing.T) {
info := &relaycommon.RelayInfo{
SendResponseCount: 1,
ClaudeConvertInfo: &relaycommon.ClaudeConvertInfo{
LastMessagesType: relaycommon.LastMessageTypeNone,
},
}
streamResp := &dto.ChatCompletionsStreamResponse{
Id: "chatcmpl_1",
Model: "gpt-test",
Choices: []dto.ChatCompletionsStreamResponseChoice{
{
Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
Content: ptrValue("hello"),
},
},
},
}
claudeResponses := StreamResponseOpenAI2Claude(streamResp, info)
require.NotEmpty(t, claudeResponses)
geminiResp := StreamResponseOpenAI2Gemini(streamResp, &relaycommon.RelayInfo{})
require.NotNil(t, geminiResp)
require.Len(t, geminiResp.Candidates, 1)
}
func ptrValue[T any](value T) *T {
return &value
}
package claudemessages
import (
"fmt"
"strings"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/constant"
"github.com/QuantumNous/new-api/dto"
relaycommon "github.com/QuantumNous/new-api/relay/common"
relaymeta "github.com/QuantumNous/new-api/service/relayconvert/internal/meta"
)
const (
webSearchMaxUsesLow = 1
webSearchMaxUsesMedium = 5
webSearchMaxUsesHigh = 10
)
type openRouterRequestReasoning struct {
Enabled bool `json:"enabled"`
Effort string `json:"effort,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
Exclude bool `json:"exclude,omitempty"`
}
func ClaudeMessagesRequestToOpenAIChat(claudeRequest dto.ClaudeRequest, info *relaycommon.RelayInfo) (*dto.GeneralOpenAIRequest, error) {
openAIRequest := dto.GeneralOpenAIRequest{
Model: claudeRequest.Model,
Temperature: claudeRequest.Temperature,
}
if claudeRequest.MaxTokens != nil {
openAIRequest.MaxTokens = common.GetPointer(*claudeRequest.MaxTokens)
}
if claudeRequest.TopP != nil {
openAIRequest.TopP = common.GetPointer(*claudeRequest.TopP)
}
if claudeRequest.TopK != nil {
openAIRequest.TopK = common.GetPointer(*claudeRequest.TopK)
}
if claudeRequest.Stream != nil {
openAIRequest.Stream = common.GetPointer(*claudeRequest.Stream)
}
isOpenRouter := relaymeta.RelayInfoChannelType(info) == constant.ChannelTypeOpenRouter
if isOpenRouter {
if effort := claudeRequest.GetEfforts(); effort != "" {
effortBytes, _ := common.Marshal(effort)
openAIRequest.Verbosity = effortBytes
}
if claudeRequest.Thinking != nil {
var reasoningConfig openRouterRequestReasoning
if claudeRequest.Thinking.Type == "enabled" {
reasoningConfig = openRouterRequestReasoning{
Enabled: true,
MaxTokens: claudeRequest.Thinking.GetBudgetTokens(),
}
} else if claudeRequest.Thinking.Type == "adaptive" {
reasoningConfig = openRouterRequestReasoning{
Enabled: true,
}
}
reasoningJSON, err := common.Marshal(reasoningConfig)
if err != nil {
return nil, fmt.Errorf("failed to marshal reasoning: %w", err)
}
openAIRequest.Reasoning = reasoningJSON
}
} else if info != nil {
thinkingSuffix := "-thinking"
if strings.HasSuffix(info.OriginModelName, thinkingSuffix) &&
!strings.HasSuffix(openAIRequest.Model, thinkingSuffix) {
openAIRequest.Model = openAIRequest.Model + thinkingSuffix
}
}
if len(claudeRequest.StopSequences) == 1 {
openAIRequest.Stop = claudeRequest.StopSequences[0]
} else if len(claudeRequest.StopSequences) > 1 {
openAIRequest.Stop = claudeRequest.StopSequences
}
tools, _ := common.Any2Type[[]dto.Tool](claudeRequest.Tools)
openAITools := make([]dto.ToolCallRequest, 0)
for _, claudeTool := range tools {
openAITool := dto.ToolCallRequest{
Type: "function",
Function: dto.FunctionRequest{
Name: claudeTool.Name,
Description: claudeTool.Description,
Parameters: claudeTool.InputSchema,
},
}
openAITools = append(openAITools, openAITool)
}
openAIRequest.Tools = openAITools
openAIMessages := make([]dto.Message, 0)
if claudeRequest.System != nil {
if claudeRequest.IsStringSystem() && claudeRequest.GetStringSystem() != "" {
openAIMessage := dto.Message{
Role: "system",
}
openAIMessage.SetStringContent(claudeRequest.GetStringSystem())
openAIMessages = append(openAIMessages, openAIMessage)
} else {
systems := claudeRequest.ParseSystem()
if len(systems) > 0 {
openAIMessage := dto.Message{
Role: "system",
}
isOpenRouterClaude := isOpenRouter && strings.HasPrefix(relaymeta.RelayInfoUpstreamModelName(info), "anthropic/claude")
if isOpenRouterClaude {
systemMediaMessages := make([]dto.MediaContent, 0, len(systems))
for _, system := range systems {
message := dto.MediaContent{
Type: "text",
Text: system.GetText(),
CacheControl: system.CacheControl,
}
systemMediaMessages = append(systemMediaMessages, message)
}
openAIMessage.SetMediaContent(systemMediaMessages)
} else {
systemStr := ""
for _, system := range systems {
if system.Text != nil {
systemStr += *system.Text
}
}
openAIMessage.SetStringContent(systemStr)
}
openAIMessages = append(openAIMessages, openAIMessage)
}
}
}
for _, claudeMessage := range claudeRequest.Messages {
openAIMessage := dto.Message{
Role: claudeMessage.Role,
}
if claudeMessage.IsStringContent() {
openAIMessage.SetStringContent(claudeMessage.GetStringContent())
} else {
content, err := claudeMessage.ParseContent()
if err != nil {
return nil, err
}
var toolCalls []dto.ToolCallRequest
mediaMessages := make([]dto.MediaContent, 0, len(content))
for _, mediaMsg := range content {
switch mediaMsg.Type {
case "text", "input_text":
message := dto.MediaContent{
Type: "text",
Text: mediaMsg.GetText(),
CacheControl: mediaMsg.CacheControl,
}
mediaMessages = append(mediaMessages, message)
case "image":
imageData := fmt.Sprintf("data:%s;base64,%s", mediaMsg.Source.MediaType, mediaMsg.Source.Data)
mediaMessage := dto.MediaContent{
Type: "image_url",
ImageUrl: &dto.MessageImageUrl{Url: imageData},
}
mediaMessages = append(mediaMessages, mediaMessage)
case "tool_use":
toolCall := dto.ToolCallRequest{
ID: mediaMsg.Id,
Type: "function",
Function: dto.FunctionRequest{
Name: mediaMsg.Name,
Arguments: requestToJSONString(mediaMsg.Input),
},
}
toolCalls = append(toolCalls, toolCall)
case "tool_result":
toolName := mediaMsg.Name
if toolName == "" {
toolName = claudeRequest.SearchToolNameByToolCallId(mediaMsg.ToolUseId)
}
oaiToolMessage := dto.Message{
Role: "tool",
Name: &toolName,
ToolCallId: mediaMsg.ToolUseId,
}
if mediaMsg.IsStringContent() {
oaiToolMessage.SetStringContent(mediaMsg.GetStringContent())
} else {
mediaContents := mediaMsg.ParseMediaContent()
encodedJSON, _ := common.Marshal(mediaContents)
oaiToolMessage.SetStringContent(string(encodedJSON))
}
openAIMessages = append(openAIMessages, oaiToolMessage)
}
}
if len(toolCalls) > 0 {
openAIMessage.SetToolCalls(toolCalls)
}
if len(mediaMessages) > 0 && len(toolCalls) == 0 {
openAIMessage.SetMediaContent(mediaMessages)
}
}
if len(openAIMessage.ParseContent()) > 0 || len(openAIMessage.ToolCalls) > 0 {
openAIMessages = append(openAIMessages, openAIMessage)
}
}
openAIRequest.Messages = openAIMessages
return &openAIRequest, nil
}
func requestToJSONString(v interface{}) string {
b, err := common.Marshal(v)
if err != nil {
return "{}"
}
return string(b)
}
package geminichat
import (
"fmt"
"strings"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/dto"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/service/relayconvert/internal/jsonutil"
relaymeta "github.com/QuantumNous/new-api/service/relayconvert/internal/meta"
)
func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatRequest, info *relaycommon.RelayInfo) (*dto.GeneralOpenAIRequest, error) {
modelName := ""
isStream := false
if info != nil {
isStream = info.IsStream
}
modelName = relaymeta.RelayInfoUpstreamModelName(info)
openaiRequest := &dto.GeneralOpenAIRequest{
Model: modelName,
Stream: common.GetPointer(isStream),
}
var messages []dto.Message
for _, content := range geminiRequest.Contents {
message := dto.Message{
Role: convertGeminiRoleToOpenAI(content.Role),
}
var mediaContents []dto.MediaContent
var toolCalls []dto.ToolCallRequest
for _, part := range content.Parts {
if part.Text != "" {
mediaContent := dto.MediaContent{
Type: "text",
Text: part.Text,
}
mediaContents = append(mediaContents, mediaContent)
} else if part.InlineData != nil {
mediaContent := dto.MediaContent{
Type: "image_url",
ImageUrl: &dto.MessageImageUrl{
Url: fmt.Sprintf("data:%s;base64,%s", part.InlineData.MimeType, part.InlineData.Data),
Detail: "auto",
MimeType: part.InlineData.MimeType,
},
}
mediaContents = append(mediaContents, mediaContent)
} else if part.FileData != nil {
mediaContent := dto.MediaContent{
Type: "image_url",
ImageUrl: &dto.MessageImageUrl{
Url: part.FileData.FileUri,
Detail: "auto",
MimeType: part.FileData.MimeType,
},
}
mediaContents = append(mediaContents, mediaContent)
} else if part.FunctionCall != nil {
toolCall := dto.ToolCallRequest{
ID: fmt.Sprintf("call_%d", len(toolCalls)+1),
Type: "function",
Function: dto.FunctionRequest{
Name: part.FunctionCall.FunctionName,
Arguments: jsonutil.ToJSONString(part.FunctionCall.Arguments),
},
}
toolCalls = append(toolCalls, toolCall)
} else if part.FunctionResponse != nil {
toolMessage := dto.Message{
Role: "tool",
ToolCallId: fmt.Sprintf("call_%d", len(toolCalls)),
}
toolMessage.SetStringContent(jsonutil.ToJSONString(part.FunctionResponse.Response))
messages = append(messages, toolMessage)
}
}
if len(toolCalls) > 0 {
message.SetToolCalls(toolCalls)
} else if len(mediaContents) == 1 && mediaContents[0].Type == "text" {
message.Content = mediaContents[0].Text
} else if len(mediaContents) > 0 {
message.SetMediaContent(mediaContents)
}
if len(message.ParseContent()) > 0 || len(message.ToolCalls) > 0 {
messages = append(messages, message)
}
}
openaiRequest.Messages = messages
if geminiRequest.GenerationConfig.Temperature != nil {
openaiRequest.Temperature = geminiRequest.GenerationConfig.Temperature
}
if geminiRequest.GenerationConfig.TopP != nil && *geminiRequest.GenerationConfig.TopP > 0 {
openaiRequest.TopP = common.GetPointer(*geminiRequest.GenerationConfig.TopP)
}
if geminiRequest.GenerationConfig.TopK != nil && *geminiRequest.GenerationConfig.TopK > 0 {
openaiRequest.TopK = common.GetPointer(int(*geminiRequest.GenerationConfig.TopK))
}
if geminiRequest.GenerationConfig.MaxOutputTokens != nil && *geminiRequest.GenerationConfig.MaxOutputTokens > 0 {
openaiRequest.MaxTokens = common.GetPointer(*geminiRequest.GenerationConfig.MaxOutputTokens)
}
if len(geminiRequest.GenerationConfig.StopSequences) > 0 {
openaiRequest.Stop = geminiRequest.GenerationConfig.StopSequences[:min(len(geminiRequest.GenerationConfig.StopSequences), 4)]
}
if geminiRequest.GenerationConfig.CandidateCount != nil && *geminiRequest.GenerationConfig.CandidateCount > 0 {
openaiRequest.N = common.GetPointer(*geminiRequest.GenerationConfig.CandidateCount)
}
if len(geminiRequest.GetTools()) > 0 {
var tools []dto.ToolCallRequest
for _, tool := range geminiRequest.GetTools() {
if tool.FunctionDeclarations == nil {
continue
}
functionDeclarations, err := common.Any2Type[[]dto.FunctionRequest](tool.FunctionDeclarations)
if err != nil {
common.SysError(fmt.Sprintf("failed to parse gemini function declarations: %v (type=%T)", err, tool.FunctionDeclarations))
continue
}
for _, function := range functionDeclarations {
openAITool := dto.ToolCallRequest{
Type: "function",
Function: dto.FunctionRequest{
Name: function.Name,
Description: function.Description,
Parameters: function.Parameters,
},
}
tools = append(tools, openAITool)
}
}
if len(tools) > 0 {
openaiRequest.Tools = tools
}
}
if geminiRequest.SystemInstructions != nil {
systemMessage := dto.Message{
Role: "system",
Content: extractTextFromGeminiParts(geminiRequest.SystemInstructions.Parts),
}
openaiRequest.Messages = append([]dto.Message{systemMessage}, openaiRequest.Messages...)
}
return openaiRequest, nil
}
func convertGeminiRoleToOpenAI(geminiRole string) string {
switch geminiRole {
case "user":
return "user"
case "model":
return "assistant"
case "function":
return "function"
default:
return "user"
}
}
func extractTextFromGeminiParts(parts []dto.GeminiPart) string {
texts := make([]string, 0)
for _, part := range parts {
if part.Text != "" {
texts = append(texts, part.Text)
}
}
return strings.Join(texts, "\n")
}
package geminichat
import (
"fmt"
"strings"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/constant"
"github.com/QuantumNous/new-api/dto"
)
func UsageFromGeminiMetadata(metadata *dto.GeminiUsageMetadata, fallbackPromptTokens int) *dto.Usage {
if metadata == nil {
if fallbackPromptTokens <= 0 {
return nil
}
usage := &dto.Usage{PromptTokens: fallbackPromptTokens}
usage.PromptTokensDetails.TextTokens = fallbackPromptTokens
return usage
}
promptTokens := metadata.PromptTokenCount + metadata.ToolUsePromptTokenCount
if promptTokens <= 0 && fallbackPromptTokens > 0 {
promptTokens = fallbackPromptTokens
}
usage := &dto.Usage{
PromptTokens: promptTokens,
CompletionTokens: metadata.CandidatesTokenCount + metadata.ThoughtsTokenCount,
TotalTokens: metadata.TotalTokenCount,
BillingUsage: dto.CloneBillingUsage(metadata.BillingUsage),
}
if usage.BillingUsage == nil {
usage.BillingUsage = dto.NewGeminiChatBillingUsage(metadata)
}
usage.CompletionTokenDetails.ReasoningTokens = metadata.ThoughtsTokenCount
usage.PromptTokensDetails.CachedTokens = metadata.CachedContentTokenCount
for _, detail := range metadata.PromptTokensDetails {
if detail.Modality == "AUDIO" {
usage.PromptTokensDetails.AudioTokens += detail.TokenCount
} else if detail.Modality == "IMAGE" {
usage.PromptTokensDetails.ImageTokens += detail.TokenCount
} else if detail.Modality == "TEXT" {
usage.PromptTokensDetails.TextTokens += detail.TokenCount
}
}
for _, detail := range metadata.ToolUsePromptTokensDetails {
if detail.Modality == "AUDIO" {
usage.PromptTokensDetails.AudioTokens += detail.TokenCount
} else if detail.Modality == "IMAGE" {
usage.PromptTokensDetails.ImageTokens += detail.TokenCount
} else if detail.Modality == "TEXT" {
usage.PromptTokensDetails.TextTokens += detail.TokenCount
}
}
for _, detail := range metadata.CandidatesTokensDetails {
switch detail.Modality {
case "IMAGE":
usage.CompletionTokenDetails.ImageTokens += detail.TokenCount
case "AUDIO":
usage.CompletionTokenDetails.AudioTokens += detail.TokenCount
case "TEXT":
usage.CompletionTokenDetails.TextTokens += detail.TokenCount
}
}
if usage.TotalTokens > 0 && usage.CompletionTokens <= 0 {
usage.CompletionTokens = usage.TotalTokens - usage.PromptTokens
}
if usage.PromptTokens > 0 && usage.PromptTokensDetails.TextTokens == 0 && usage.PromptTokensDetails.AudioTokens == 0 {
usage.PromptTokensDetails.TextTokens = usage.PromptTokens
}
return usage
}
func ResponseGeminiChat2OpenAI(id string, created int64, response *dto.GeminiChatResponse) *dto.OpenAITextResponse {
fullTextResponse := dto.OpenAITextResponse{
Id: id,
Object: "chat.completion",
Created: created,
Choices: make([]dto.OpenAITextResponseChoice, 0, len(response.Candidates)),
}
isToolCall := false
for _, candidate := range response.Candidates {
choice := dto.OpenAITextResponseChoice{
Index: int(candidate.Index),
Message: dto.Message{
Role: "assistant",
Content: "",
},
FinishReason: constant.FinishReasonStop,
}
if len(candidate.Content.Parts) > 0 {
var content strings.Builder
var inlineGrow int
for _, part := range candidate.Content.Parts {
if part.InlineData != nil {
inlineGrow += len(part.InlineData.MimeType) + len(part.InlineData.Data) + 32
}
}
if inlineGrow > 0 {
content.Grow(inlineGrow)
}
appended := 0
writeSep := func() {
if appended > 0 {
content.WriteByte('\n')
}
appended++
}
var toolCalls []dto.ToolCallResponse
for _, part := range candidate.Content.Parts {
if part.InlineData != nil {
if strings.HasPrefix(part.InlineData.MimeType, "image") {
writeSep()
content.WriteString("![image](data:")
content.WriteString(part.InlineData.MimeType)
content.WriteString(";base64,")
content.WriteString(part.InlineData.Data)
content.WriteByte(')')
} else {
writeSep()
content.WriteString("[media](data:")
content.WriteString(part.InlineData.MimeType)
content.WriteString(";base64,")
content.WriteString(part.InlineData.Data)
content.WriteByte(')')
}
} else if part.FunctionCall != nil {
choice.FinishReason = constant.FinishReasonToolCalls
if call := geminiResponseToolCall(&part); call != nil {
toolCalls = append(toolCalls, *call)
}
} else if part.Thought {
choice.Message.ReasoningContent = &part.Text
} else {
if part.ExecutableCode != nil {
writeSep()
content.WriteString("```")
content.WriteString(part.ExecutableCode.Language)
content.WriteByte('\n')
content.WriteString(part.ExecutableCode.Code)
content.WriteString("\n```")
} else if part.CodeExecutionResult != nil {
writeSep()
content.WriteString("```output\n")
content.WriteString(part.CodeExecutionResult.Output)
content.WriteString("\n```")
} else if part.Text != "\n" {
writeSep()
content.WriteString(part.Text)
}
}
}
if len(toolCalls) > 0 {
choice.Message.SetToolCalls(toolCalls)
isToolCall = true
}
choice.Message.SetStringContent(content.String())
}
if candidate.FinishReason != nil {
switch *candidate.FinishReason {
case "STOP":
choice.FinishReason = constant.FinishReasonStop
case "MAX_TOKENS":
choice.FinishReason = constant.FinishReasonLength
case "SAFETY", "RECITATION", "BLOCKLIST", "PROHIBITED_CONTENT", "SPII", "OTHER":
choice.FinishReason = constant.FinishReasonContentFilter
default:
choice.FinishReason = constant.FinishReasonContentFilter
}
}
if isToolCall {
choice.FinishReason = constant.FinishReasonToolCalls
}
fullTextResponse.Choices = append(fullTextResponse.Choices, choice)
}
return &fullTextResponse
}
func StreamResponseGeminiChat2OpenAI(geminiResponse *dto.GeminiChatResponse) (*dto.ChatCompletionsStreamResponse, bool) {
choices := make([]dto.ChatCompletionsStreamResponseChoice, 0, len(geminiResponse.Candidates))
isStop := false
for _, candidate := range geminiResponse.Candidates {
if candidate.FinishReason != nil && *candidate.FinishReason == "STOP" {
isStop = true
candidate.FinishReason = nil
}
choice := dto.ChatCompletionsStreamResponseChoice{
Index: int(candidate.Index),
Delta: dto.ChatCompletionsStreamResponseChoiceDelta{},
}
var content strings.Builder
var inlineGrow int
for _, part := range candidate.Content.Parts {
if part.InlineData != nil {
inlineGrow += len(part.InlineData.MimeType) + len(part.InlineData.Data) + 32
}
}
if inlineGrow > 0 {
content.Grow(inlineGrow)
}
appended := 0
writeSep := func() {
if appended > 0 {
content.WriteByte('\n')
}
appended++
}
isTools := false
isThought := false
if candidate.FinishReason != nil {
switch *candidate.FinishReason {
case "STOP":
choice.FinishReason = &constant.FinishReasonStop
case "MAX_TOKENS":
choice.FinishReason = &constant.FinishReasonLength
case "SAFETY", "RECITATION", "BLOCKLIST", "PROHIBITED_CONTENT", "SPII", "OTHER":
choice.FinishReason = &constant.FinishReasonContentFilter
default:
choice.FinishReason = &constant.FinishReasonContentFilter
}
}
for _, part := range candidate.Content.Parts {
if part.InlineData != nil {
if strings.HasPrefix(part.InlineData.MimeType, "image") {
writeSep()
content.WriteString("![image](data:")
content.WriteString(part.InlineData.MimeType)
content.WriteString(";base64,")
content.WriteString(part.InlineData.Data)
content.WriteByte(')')
}
} else if part.FunctionCall != nil {
isTools = true
if call := geminiResponseToolCall(&part); call != nil {
call.SetIndex(len(choice.Delta.ToolCalls))
choice.Delta.ToolCalls = append(choice.Delta.ToolCalls, *call)
}
} else if part.Thought {
isThought = true
writeSep()
content.WriteString(part.Text)
} else {
if part.ExecutableCode != nil {
writeSep()
content.WriteString("```")
content.WriteString(part.ExecutableCode.Language)
content.WriteByte('\n')
content.WriteString(part.ExecutableCode.Code)
content.WriteString("\n```\n")
} else if part.CodeExecutionResult != nil {
writeSep()
content.WriteString("```output\n")
content.WriteString(part.CodeExecutionResult.Output)
content.WriteString("\n```\n")
} else if part.Text != "\n" {
writeSep()
content.WriteString(part.Text)
}
}
}
if isThought {
choice.Delta.SetReasoningContent(content.String())
} else {
choice.Delta.SetContentString(content.String())
}
if isTools {
choice.FinishReason = &constant.FinishReasonToolCalls
}
choices = append(choices, choice)
}
response := dto.ChatCompletionsStreamResponse{
Object: "chat.completion.chunk",
Choices: choices,
}
return &response, isStop
}
func geminiResponseToolCall(item *dto.GeminiPart) *dto.ToolCallResponse {
argsBytes, err := common.Marshal(item.FunctionCall.Arguments)
if err != nil {
return nil
}
return &dto.ToolCallResponse{
ID: fmt.Sprintf("call_%s", common.GetUUID()),
Type: "function",
Function: dto.FunctionResponse{
Arguments: string(argsBytes),
Name: item.FunctionCall.FunctionName,
},
}
}
package jsonutil
import (
"fmt"
"github.com/QuantumNous/new-api/common"
)
func ToJSONString(v interface{}) string {
bytes, err := common.Marshal(v)
if err != nil {
return fmt.Sprintf("%v", v)
}
return string(bytes)
}
package relayconvert
package matcher
import (
"regexp"
......@@ -7,7 +7,7 @@ import (
var compiledRegexCache sync.Map // map[string]*regexp.Regexp
func matchAnyRegex(patterns []string, s string) bool {
func MatchAnyRegex(patterns []string, s string) bool {
if len(patterns) == 0 || s == "" {
return false
}
......
package media
import (
"errors"
"sync"
"github.com/QuantumNous/new-api/types"
"github.com/gin-gonic/gin"
)
type MediaResolver struct {
GetBase64Data func(c *gin.Context, source types.FileSource, reason ...string) (string, string, error)
DecodeBase64FileData func(base64String string) (string, string, error)
}
var (
mediaResolverMu sync.RWMutex
mediaResolver MediaResolver
)
func SetMediaResolver(resolver MediaResolver) {
mediaResolverMu.Lock()
defer mediaResolverMu.Unlock()
mediaResolver = resolver
}
func ResolveBase64Data(c *gin.Context, source types.FileSource, reason ...string) (string, string, error) {
mediaResolverMu.RLock()
resolver := mediaResolver.GetBase64Data
mediaResolverMu.RUnlock()
if resolver == nil {
return "", "", errors.New("relayconvert media resolver is not configured")
}
return resolver(c, source, reason...)
}
func DecodeBase64FileData(base64String string) (string, string, error) {
mediaResolverMu.RLock()
resolver := mediaResolver.DecodeBase64FileData
mediaResolverMu.RUnlock()
if resolver == nil {
return "", "", errors.New("relayconvert media resolver is not configured")
}
return resolver(base64String)
}
package meta
import relaycommon "github.com/QuantumNous/new-api/relay/common"
func RelayInfoChannelType(info *relaycommon.RelayInfo) int {
if info == nil || info.ChannelMeta == nil {
return 0
}
return info.ChannelType
}
func RelayInfoUpstreamModelName(info *relaycommon.RelayInfo) string {
if info == nil || info.ChannelMeta == nil {
return ""
}
return info.UpstreamModelName
}
package oaichat
import (
"testing"
"github.com/QuantumNous/new-api/dto"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestResponseOpenAI2ClaudeToolUseInputIsObject(t *testing.T) {
tests := []struct {
name string
args string
want map[string]interface{}
}{
{name: "object", args: `{"q":"x"}`, want: map[string]interface{}{"q": "x"}},
{name: "empty", args: "", want: map[string]interface{}{}},
{name: "invalid", args: "{", want: map[string]interface{}{}},
{name: "null", args: "null", want: map[string]interface{}{}},
{name: "array", args: `["x"]`, want: map[string]interface{}{}},
{name: "string", args: `"x"`, want: map[string]interface{}{}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
msg := dto.Message{Role: "assistant"}
msg.SetToolCalls([]dto.ToolCallRequest{
{
ID: "call_1",
Type: "function",
Function: dto.FunctionRequest{
Name: "lookup",
Arguments: tt.args,
},
},
})
resp := ResponseOpenAI2Claude(&dto.OpenAITextResponse{
Id: "chatcmpl_1",
Model: "gpt-test",
Choices: []dto.OpenAITextResponseChoice{
{Message: msg, FinishReason: "tool_calls"},
},
}, nil)
require.Len(t, resp.Content, 1)
assert.Equal(t, "tool_use", resp.Content[0].Type)
assert.Equal(t, tt.want, resp.Content[0].Input)
})
}
}
func TestResponseOpenAI2ClaudeUsageCarriesOpenAIBillingUsage(t *testing.T) {
resp := ResponseOpenAI2Claude(&dto.OpenAITextResponse{
Id: "chatcmpl_1",
Model: "gpt-test",
Choices: []dto.OpenAITextResponseChoice{
{Message: dto.Message{Role: "assistant", Content: "hello"}, FinishReason: "stop"},
},
Usage: dto.Usage{
PromptTokens: 11,
CompletionTokens: 5,
TotalTokens: 16,
},
}, nil)
require.NotNil(t, resp.Usage)
assert.Equal(t, 11, resp.Usage.InputTokens)
assert.Equal(t, 5, resp.Usage.OutputTokens)
require.NotNil(t, resp.Usage.BillingUsage)
require.NotNil(t, resp.Usage.BillingUsage.OpenAIUsage)
assert.Equal(t, dto.BillingUsageSourceOAIChat, resp.Usage.BillingUsage.Source)
assert.Equal(t, dto.BillingUsageSemanticOpenAI, resp.Usage.BillingUsage.Semantic)
assert.Equal(t, 11, resp.Usage.BillingUsage.OpenAIUsage.PromptTokens)
assert.Equal(t, 5, resp.Usage.BillingUsage.OpenAIUsage.CompletionTokens)
assert.Equal(t, 16, resp.Usage.BillingUsage.OpenAIUsage.TotalTokens)
assert.Nil(t, resp.Usage.BillingUsage.OpenAIUsage.BillingUsage)
}
func TestStreamResponseOpenAI2ClaudeClosesTextThinkingAndToolBlocks(t *testing.T) {
info := &relaycommon.RelayInfo{
ClaudeConvertInfo: &relaycommon.ClaudeConvertInfo{
LastMessagesType: relaycommon.LastMessageTypeNone,
},
}
info.SendResponseCount = 1
textResponses := StreamResponseOpenAI2Claude(&dto.ChatCompletionsStreamResponse{
Id: "chatcmpl_1",
Model: "gpt-test",
Choices: []dto.ChatCompletionsStreamResponseChoice{
{
Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
Content: ptr("hello"),
},
},
},
}, info)
require.Len(t, textResponses, 3)
assert.Equal(t, "message_start", textResponses[0].Type)
assert.Equal(t, "content_block_start", textResponses[1].Type)
assert.Equal(t, 0, textResponses[1].GetIndex())
assert.Equal(t, "content_block_delta", textResponses[2].Type)
info.SendResponseCount = 2
thinkingResponses := StreamResponseOpenAI2Claude(&dto.ChatCompletionsStreamResponse{
Id: "chatcmpl_1",
Model: "gpt-test",
Choices: []dto.ChatCompletionsStreamResponseChoice{
{
Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
ReasoningContent: ptr("thinking"),
},
},
},
}, info)
require.Len(t, thinkingResponses, 3)
assert.Equal(t, "content_block_stop", thinkingResponses[0].Type)
assert.Equal(t, 0, thinkingResponses[0].GetIndex())
assert.Equal(t, "content_block_start", thinkingResponses[1].Type)
assert.Equal(t, 1, thinkingResponses[1].GetIndex())
assert.Equal(t, "thinking", thinkingResponses[1].ContentBlock.Type)
assert.Equal(t, "content_block_delta", thinkingResponses[2].Type)
info.SendResponseCount = 3
toolResponses := StreamResponseOpenAI2Claude(&dto.ChatCompletionsStreamResponse{
Id: "chatcmpl_1",
Model: "gpt-test",
Choices: []dto.ChatCompletionsStreamResponseChoice{
{
Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
ToolCalls: []dto.ToolCallResponse{
{
Index: ptr(0),
ID: "call_1",
Type: "function",
Function: dto.FunctionResponse{
Name: "lookup",
Arguments: `{"q":"x"}`,
},
},
},
},
},
},
}, info)
require.Len(t, toolResponses, 3)
assert.Equal(t, "content_block_stop", toolResponses[0].Type)
assert.Equal(t, 1, toolResponses[0].GetIndex())
assert.Equal(t, "content_block_start", toolResponses[1].Type)
assert.Equal(t, 2, toolResponses[1].GetIndex())
assert.Equal(t, "tool_use", toolResponses[1].ContentBlock.Type)
assert.Equal(t, "content_block_delta", toolResponses[2].Type)
info.SendResponseCount = 4
finishResponses := StreamResponseOpenAI2Claude(&dto.ChatCompletionsStreamResponse{
Id: "chatcmpl_1",
Model: "gpt-test",
Choices: []dto.ChatCompletionsStreamResponseChoice{
{FinishReason: ptr("tool_calls")},
},
Usage: &dto.Usage{
PromptTokens: 7,
CompletionTokens: 3,
TotalTokens: 10,
},
}, info)
require.Len(t, finishResponses, 3)
assert.Equal(t, "content_block_stop", finishResponses[0].Type)
assert.Equal(t, 2, finishResponses[0].GetIndex())
assert.Equal(t, "message_delta", finishResponses[1].Type)
assert.Equal(t, "tool_use", *finishResponses[1].Delta.StopReason)
require.NotNil(t, finishResponses[1].Usage)
require.NotNil(t, finishResponses[1].Usage.BillingUsage)
require.NotNil(t, finishResponses[1].Usage.BillingUsage.OpenAIUsage)
assert.Equal(t, 7, finishResponses[1].Usage.BillingUsage.OpenAIUsage.PromptTokens)
assert.Equal(t, 3, finishResponses[1].Usage.BillingUsage.OpenAIUsage.CompletionTokens)
assert.Equal(t, "message_stop", finishResponses[2].Type)
}
func TestNormalizeCacheCreationSplit(t *testing.T) {
cache5m, cache1h := NormalizeCacheCreationSplit(10, 3, 2)
assert.Equal(t, 8, cache5m)
assert.Equal(t, 2, cache1h)
cache5m, cache1h = NormalizeCacheCreationSplit(3, 5, 1)
assert.Equal(t, 5, cache5m)
assert.Equal(t, 1, cache1h)
}
func ptr[T any](value T) *T {
return &value
}
package oaichat
import (
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/dto"
relaycommon "github.com/QuantumNous/new-api/relay/common"
)
// ResponseOpenAI2Gemini 将 OpenAI 响应转换为 Gemini 格式
func ResponseOpenAI2Gemini(openAIResponse *dto.OpenAITextResponse, info *relaycommon.RelayInfo) *dto.GeminiChatResponse {
totalTokens := openAIResponse.TotalTokens
if totalTokens == 0 {
totalTokens = openAIResponse.PromptTokens + openAIResponse.CompletionTokens
}
geminiResponse := &dto.GeminiChatResponse{
Candidates: make([]dto.GeminiChatCandidate, 0, len(openAIResponse.Choices)),
HasUsageMetadata: true,
UsageMetadata: dto.GeminiUsageMetadata{
PromptTokenCount: openAIResponse.PromptTokens,
CandidatesTokenCount: openAIResponse.CompletionTokens,
TotalTokenCount: totalTokens,
BillingUsage: openAIBillingUsageFromUsage(&openAIResponse.Usage),
},
}
if metadata, ok := geminiBillingMetadataFromOpenAIUsage(&openAIResponse.Usage); ok {
geminiResponse.UsageMetadata = metadata
}
for _, choice := range openAIResponse.Choices {
candidate := dto.GeminiChatCandidate{
Index: int64(choice.Index),
SafetyRatings: []dto.GeminiChatSafetyRating{},
}
// 设置结束原因
var finishReason string
switch choice.FinishReason {
case "stop":
finishReason = "STOP"
case "length":
finishReason = "MAX_TOKENS"
case "content_filter":
finishReason = "SAFETY"
case "tool_calls":
finishReason = "STOP"
default:
finishReason = "STOP"
}
candidate.FinishReason = &finishReason
// 转换消息内容
content := dto.GeminiChatContent{
Role: "model",
Parts: make([]dto.GeminiPart, 0),
}
textContent := choice.Message.StringContent()
if textContent != "" {
part := dto.GeminiPart{
Text: textContent,
}
content.Parts = append(content.Parts, part)
}
toolCalls := choice.Message.ParseToolCalls()
for _, toolCall := range toolCalls {
var args map[string]interface{}
if toolCall.Function.Arguments != "" {
if err := common.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil {
args = map[string]interface{}{"arguments": toolCall.Function.Arguments}
}
} else {
args = make(map[string]interface{})
}
part := dto.GeminiPart{
FunctionCall: &dto.FunctionCall{
FunctionName: toolCall.Function.Name,
Arguments: args,
},
}
content.Parts = append(content.Parts, part)
}
candidate.Content = content
geminiResponse.Candidates = append(geminiResponse.Candidates, candidate)
}
return geminiResponse
}
// StreamResponseOpenAI2Gemini 将 OpenAI 流式响应转换为 Gemini 格式
func StreamResponseOpenAI2Gemini(openAIResponse *dto.ChatCompletionsStreamResponse, info *relaycommon.RelayInfo) *dto.GeminiChatResponse {
// 检查是否有实际内容或结束标志
hasContent := false
hasFinishReason := false
for _, choice := range openAIResponse.Choices {
if len(choice.Delta.GetContentString()) > 0 || (choice.Delta.ToolCalls != nil && len(choice.Delta.ToolCalls) > 0) {
hasContent = true
}
if choice.FinishReason != nil {
hasFinishReason = true
}
}
// 如果没有实际内容且没有结束标志,跳过。主要针对 openai 流响应开头的空数据
if !hasContent && !hasFinishReason {
return nil
}
estimatePromptTokens := 0
if info != nil {
estimatePromptTokens = info.GetEstimatePromptTokens()
}
geminiResponse := &dto.GeminiChatResponse{
Candidates: make([]dto.GeminiChatCandidate, 0, len(openAIResponse.Choices)),
HasUsageMetadata: true,
UsageMetadata: dto.GeminiUsageMetadata{
PromptTokenCount: estimatePromptTokens,
CandidatesTokenCount: 0, // 流式响应中可能没有完整的 usage 信息
TotalTokenCount: estimatePromptTokens,
},
}
if openAIResponse.Usage != nil {
geminiResponse.UsageMetadata.PromptTokenCount = openAIResponse.Usage.PromptTokens
geminiResponse.UsageMetadata.CandidatesTokenCount = openAIResponse.Usage.CompletionTokens
geminiResponse.UsageMetadata.TotalTokenCount = openAIResponse.Usage.TotalTokens
geminiResponse.UsageMetadata.BillingUsage = openAIBillingUsageFromUsage(openAIResponse.Usage)
if metadata, ok := geminiBillingMetadataFromOpenAIUsage(openAIResponse.Usage); ok {
geminiResponse.UsageMetadata = metadata
}
}
for _, choice := range openAIResponse.Choices {
candidate := dto.GeminiChatCandidate{
Index: int64(choice.Index),
SafetyRatings: []dto.GeminiChatSafetyRating{},
}
// 设置结束原因
if choice.FinishReason != nil {
var finishReason string
switch *choice.FinishReason {
case "stop":
finishReason = "STOP"
case "length":
finishReason = "MAX_TOKENS"
case "content_filter":
finishReason = "SAFETY"
case "tool_calls":
finishReason = "STOP"
default:
finishReason = "STOP"
}
candidate.FinishReason = &finishReason
}
// 转换消息内容
content := dto.GeminiChatContent{
Role: "model",
Parts: make([]dto.GeminiPart, 0),
}
// 处理工具调用
if choice.Delta.ToolCalls != nil {
for _, toolCall := range choice.Delta.ToolCalls {
// 解析参数
var args map[string]interface{}
if toolCall.Function.Arguments != "" {
if err := common.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil {
args = map[string]interface{}{"arguments": toolCall.Function.Arguments}
}
} else {
args = make(map[string]interface{})
}
part := dto.GeminiPart{
FunctionCall: &dto.FunctionCall{
FunctionName: toolCall.Function.Name,
Arguments: args,
},
}
content.Parts = append(content.Parts, part)
}
} else {
// 处理文本内容
textContent := choice.Delta.GetContentString()
if textContent != "" {
part := dto.GeminiPart{
Text: textContent,
}
content.Parts = append(content.Parts, part)
}
}
candidate.Content = content
geminiResponse.Candidates = append(geminiResponse.Candidates, candidate)
}
return geminiResponse
}
func geminiBillingMetadataFromOpenAIUsage(usage *dto.Usage) (dto.GeminiUsageMetadata, bool) {
if usage == nil || usage.BillingUsage == nil || usage.BillingUsage.GeminiUsageMetadata == nil {
return dto.GeminiUsageMetadata{}, false
}
if usage.BillingUsage.Source != dto.BillingUsageSourceGeminiChat && usage.BillingUsage.Semantic != dto.BillingUsageSemanticGemini {
return dto.GeminiUsageMetadata{}, false
}
billingUsage := dto.CloneBillingUsage(usage.BillingUsage)
if billingUsage == nil || billingUsage.GeminiUsageMetadata == nil {
return dto.GeminiUsageMetadata{}, false
}
return *billingUsage.GeminiUsageMetadata, true
}
func openAIBillingUsageFromUsage(usage *dto.Usage) *dto.BillingUsage {
if usage == nil {
return nil
}
if existingBillingUsage := dto.CloneBillingUsage(usage.BillingUsage); existingBillingUsage != nil && existingBillingUsage.OpenAIUsage != nil {
if existingBillingUsage.Source == dto.BillingUsageSourceOAIChat ||
existingBillingUsage.Source == dto.BillingUsageSourceOAIResponses ||
existingBillingUsage.Semantic == dto.BillingUsageSemanticOpenAI {
return existingBillingUsage
}
}
return dto.NewOpenAIChatBillingUsage(usage)
}
package oaichat
import (
"testing"
"github.com/QuantumNous/new-api/dto"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestResponseOpenAI2GeminiMapsTextToolFinishReasonAndUsage(t *testing.T) {
msg := dto.Message{
Role: "assistant",
Content: "hello",
}
msg.SetToolCalls([]dto.ToolCallRequest{
{
ID: "call_1",
Type: "function",
Function: dto.FunctionRequest{
Name: "lookup",
Arguments: `{"q":"x"}`,
},
},
})
resp := ResponseOpenAI2Gemini(&dto.OpenAITextResponse{
Model: "gpt-test",
Choices: []dto.OpenAITextResponseChoice{
{
Index: 2,
Message: msg,
FinishReason: "length",
},
},
Usage: dto.Usage{
PromptTokens: 11,
CompletionTokens: 5,
TotalTokens: 16,
},
}, nil)
assert.Equal(t, 11, resp.UsageMetadata.PromptTokenCount)
assert.Equal(t, 5, resp.UsageMetadata.CandidatesTokenCount)
assert.Equal(t, 16, resp.UsageMetadata.TotalTokenCount)
require.NotNil(t, resp.UsageMetadata.BillingUsage)
require.NotNil(t, resp.UsageMetadata.BillingUsage.OpenAIUsage)
assert.Equal(t, dto.BillingUsageSourceOAIChat, resp.UsageMetadata.BillingUsage.Source)
assert.Equal(t, dto.BillingUsageSemanticOpenAI, resp.UsageMetadata.BillingUsage.Semantic)
assert.Equal(t, 11, resp.UsageMetadata.BillingUsage.OpenAIUsage.PromptTokens)
assert.Equal(t, 5, resp.UsageMetadata.BillingUsage.OpenAIUsage.CompletionTokens)
assert.Equal(t, 16, resp.UsageMetadata.BillingUsage.OpenAIUsage.TotalTokens)
assert.Nil(t, resp.UsageMetadata.BillingUsage.OpenAIUsage.BillingUsage)
require.Len(t, resp.Candidates, 1)
assert.Equal(t, int64(2), resp.Candidates[0].Index)
require.NotNil(t, resp.Candidates[0].FinishReason)
assert.Equal(t, "MAX_TOKENS", *resp.Candidates[0].FinishReason)
require.Len(t, resp.Candidates[0].Content.Parts, 2)
assert.Equal(t, "hello", resp.Candidates[0].Content.Parts[0].Text)
require.NotNil(t, resp.Candidates[0].Content.Parts[1].FunctionCall)
assert.Equal(t, "lookup", resp.Candidates[0].Content.Parts[1].FunctionCall.FunctionName)
assert.Equal(t, map[string]interface{}{"q": "x"}, resp.Candidates[0].Content.Parts[1].FunctionCall.Arguments)
}
func TestStreamResponseOpenAI2GeminiMapsToolCallFinishReasonAndUsage(t *testing.T) {
resp := StreamResponseOpenAI2Gemini(&dto.ChatCompletionsStreamResponse{
Choices: []dto.ChatCompletionsStreamResponseChoice{
{
Index: 1,
FinishReason: geminiRespPtr("tool_calls"),
Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
ToolCalls: []dto.ToolCallResponse{
{
Type: "function",
Function: dto.FunctionResponse{
Name: "lookup",
Arguments: `{"q":"x"}`,
},
},
},
},
},
},
Usage: &dto.Usage{
PromptTokens: 13,
CompletionTokens: 8,
TotalTokens: 21,
},
}, &relaycommon.RelayInfo{})
require.NotNil(t, resp)
assert.Equal(t, 13, resp.UsageMetadata.PromptTokenCount)
assert.Equal(t, 8, resp.UsageMetadata.CandidatesTokenCount)
assert.Equal(t, 21, resp.UsageMetadata.TotalTokenCount)
require.NotNil(t, resp.UsageMetadata.BillingUsage)
require.NotNil(t, resp.UsageMetadata.BillingUsage.OpenAIUsage)
assert.Equal(t, 13, resp.UsageMetadata.BillingUsage.OpenAIUsage.PromptTokens)
assert.Equal(t, 8, resp.UsageMetadata.BillingUsage.OpenAIUsage.CompletionTokens)
require.Len(t, resp.Candidates, 1)
assert.Equal(t, int64(1), resp.Candidates[0].Index)
require.NotNil(t, resp.Candidates[0].FinishReason)
assert.Equal(t, "STOP", *resp.Candidates[0].FinishReason)
require.Len(t, resp.Candidates[0].Content.Parts, 1)
require.NotNil(t, resp.Candidates[0].Content.Parts[0].FunctionCall)
assert.Equal(t, "lookup", resp.Candidates[0].Content.Parts[0].FunctionCall.FunctionName)
assert.Equal(t, map[string]interface{}{"q": "x"}, resp.Candidates[0].Content.Parts[0].FunctionCall.Arguments)
}
func geminiRespPtr[T any](value T) *T {
return &value
}
package relayconvert
package oaichat
import "github.com/QuantumNous/new-api/setting/model_setting"
import (
"github.com/QuantumNous/new-api/service/relayconvert/internal/matcher"
"github.com/QuantumNous/new-api/setting/model_setting"
)
func ShouldChatCompletionsUseResponsesPolicy(policy model_setting.ChatCompletionsToResponsesPolicy, channelID int, channelType int, model string) bool {
if !policy.IsChannelEnabled(channelID, channelType) {
return false
}
return matchAnyRegex(policy.ModelPatterns, model)
return matcher.MatchAnyRegex(policy.ModelPatterns, model)
}
func ShouldChatCompletionsUseResponsesGlobal(channelID int, channelType int, model string) bool {
......
package oaichat
import (
"testing"
"github.com/QuantumNous/new-api/dto"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/tidwall/gjson"
)
func TestChatCompletionsRequestToResponsesRequestInstructionsAndTools(t *testing.T) {
req := &dto.GeneralOpenAIRequest{
Model: "gpt-test",
N: lo.ToPtr(1),
Messages: []dto.Message{
{Role: "system", Content: "system rules"},
{Role: "developer", Content: "developer rules"},
{Role: "user", Content: []any{
map[string]any{"type": "text", "text": "look"},
map[string]any{"type": "image_url", "image_url": map[string]any{"url": "https://example.test/a.png"}},
}},
assistantMessageWithTool("partial text", "call_1", "lookup", `{"q":"x"}`),
{Role: "tool", ToolCallId: "call_1", Content: "tool result"},
},
}
got, err := ChatCompletionsRequestToResponsesRequest(req)
require.NoError(t, err)
assert.Equal(t, "gpt-test", got.Model)
assert.Equal(t, `"system rules\n\ndeveloper rules"`, string(got.Instructions))
assert.Equal(t, "input_image", gjson.GetBytes(got.Input, "0.content.1.type").String())
assert.Equal(t, "function_call", gjson.GetBytes(got.Input, "2.type").String())
assert.Equal(t, "call_1", gjson.GetBytes(got.Input, "2.call_id").String())
assert.Equal(t, "function_call_output", gjson.GetBytes(got.Input, "3.type").String())
}
func TestChatCompletionsRequestToResponsesRequestRejectsMultipleChoices(t *testing.T) {
_, err := ChatCompletionsRequestToResponsesRequest(&dto.GeneralOpenAIRequest{
Model: "gpt-test",
N: lo.ToPtr(2),
})
require.Error(t, err)
assert.Contains(t, err.Error(), "n>1")
}
func assistantMessageWithTool(content string, id string, name string, args string) dto.Message {
msg := dto.Message{Role: "assistant", Content: content}
msg.SetToolCalls([]dto.ToolCallRequest{
{
ID: id,
Type: "function",
Function: dto.FunctionRequest{
Name: name,
Arguments: args,
},
},
})
return msg
}
package oaichat
import (
"errors"
"fmt"
"strings"
"time"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/dto"
)
const (
chatFinishReasonLength = "length"
chatFinishReasonContentFilter = "content_filter"
responsesEventCreated = "response.created"
responsesEventCompleted = "response.completed"
responsesEventIncomplete = "response.incomplete"
responsesEventOutputTextDelta = "response.output_text.delta"
responsesEventOutputItemAdded = "response.output_item.added"
responsesEventOutputItemDone = "response.output_item.done"
responsesEventFunctionArgsDelta = "response.function_call_arguments.delta"
responsesEventFunctionArgsDone = "response.function_call_arguments.done"
responsesEventReasoningSummaryDelta = "response.reasoning_summary_text.delta"
responsesEventReasoningSummaryDone = "response.reasoning_summary_text.done"
responsesOutputTypeFunctionCall = "function_call"
responsesOutputTypeMessage = "message"
responsesOutputTypeReasoning = "reasoning"
responsesIncompleteReasonContentFilter = "content_filter"
responsesIncompleteReasonMaxTokens = "max_output_tokens"
)
func ChatCompletionsResponseToResponsesResponse(resp *dto.OpenAITextResponse, id string) (*dto.OpenAIResponsesResponse, *dto.Usage, error) {
if resp == nil {
return nil, nil, errors.New("response is nil")
}
usage := UsageFromChatUsage(&resp.Usage)
out := &dto.OpenAIResponsesResponse{
ID: id,
Object: "response",
CreatedAt: chatCreatedAt(resp.Created),
Status: []byte(`"completed"`),
Model: resp.Model,
Output: make([]dto.ResponsesOutput, 0),
Usage: usage,
}
if len(resp.Choices) == 0 {
return out, usage, nil
}
choice := resp.Choices[0]
if status, details := ResponsesStatusFromChatFinishReason(choice.FinishReason); status != "" {
out.Status = []byte(fmt.Sprintf("%q", status))
out.IncompleteDetails = details
}
if text := choice.Message.StringContent(); text != "" {
out.Output = append(out.Output, dto.ResponsesOutput{
Type: responsesOutputTypeMessage,
ID: fmt.Sprintf("%s_msg_0", id),
Status: responseOutputStatus(out),
Role: "assistant",
Content: []dto.ResponsesOutputContent{
{
Type: "output_text",
Text: text,
Annotations: []interface{}{},
},
},
})
}
if reasoning := choice.Message.GetReasoningContent(); reasoning != "" {
out.Output = append(out.Output, dto.ResponsesOutput{
Type: responsesOutputTypeReasoning,
ID: fmt.Sprintf("%s_reasoning_0", id),
Status: responseOutputStatus(out),
Content: []dto.ResponsesOutputContent{
{
Type: "summary_text",
Text: reasoning,
},
},
})
}
for i, toolCall := range choice.Message.ParseToolCalls() {
toolOutput, err := chatToolCallToResponsesOutput(toolCall, id, i, responseOutputStatus(out))
if err != nil {
return nil, nil, err
}
out.Output = append(out.Output, toolOutput)
}
return out, usage, nil
}
func ResponsesStatusFromChatFinishReason(finishReason string) (string, *dto.IncompleteDetails) {
switch strings.TrimSpace(finishReason) {
case chatFinishReasonLength:
return "incomplete", &dto.IncompleteDetails{Reason: responsesIncompleteReasonMaxTokens}
case chatFinishReasonContentFilter:
return "incomplete", &dto.IncompleteDetails{Reason: responsesIncompleteReasonContentFilter}
default:
return "completed", nil
}
}
func UsageFromChatUsage(src *dto.Usage) *dto.Usage {
usage := &dto.Usage{}
if src == nil {
return usage
}
usage.UsageSemantic = src.UsageSemantic
usage.UsageSource = src.UsageSource
usage.BillingUsage = dto.CloneBillingUsage(src.BillingUsage)
if usage.BillingUsage == nil {
usage.BillingUsage = dto.NewOpenAIChatBillingUsage(src)
}
usage.Cost = src.Cost
if src.PromptTokens != 0 {
usage.PromptTokens = src.PromptTokens
usage.InputTokens = src.PromptTokens
}
if src.CompletionTokens != 0 {
usage.CompletionTokens = src.CompletionTokens
usage.OutputTokens = src.CompletionTokens
}
if src.TotalTokens != 0 {
usage.TotalTokens = src.TotalTokens
} else {
usage.TotalTokens = usage.InputTokens + usage.OutputTokens
}
if src.PromptTokensDetails.CachedTokens != 0 ||
src.PromptTokensDetails.ImageTokens != 0 ||
src.PromptTokensDetails.AudioTokens != 0 ||
src.PromptTokensDetails.CachedCreationTokens != 0 ||
src.PromptTokensDetails.TextTokens != 0 {
details := src.PromptTokensDetails
usage.InputTokensDetails = &details
}
if src.CompletionTokenDetails.ReasoningTokens != 0 ||
src.CompletionTokenDetails.TextTokens != 0 ||
src.CompletionTokenDetails.AudioTokens != 0 ||
src.CompletionTokenDetails.ImageTokens != 0 {
usage.CompletionTokenDetails = src.CompletionTokenDetails
}
usage.ClaudeCacheCreation5mTokens = src.ClaudeCacheCreation5mTokens
usage.ClaudeCacheCreation1hTokens = src.ClaudeCacheCreation1hTokens
return usage
}
func responseOutputStatus(resp *dto.OpenAIResponsesResponse) string {
if resp == nil || responseStatusString(resp) != "incomplete" {
return "completed"
}
return "incomplete"
}
func responseStatusString(resp *dto.OpenAIResponsesResponse) string {
if resp == nil || len(resp.Status) == 0 {
return ""
}
var status string
_ = common.Unmarshal(resp.Status, &status)
return strings.TrimSpace(status)
}
func chatToolCallToResponsesOutput(toolCall dto.ToolCallRequest, responseID string, index int, status string) (dto.ResponsesOutput, error) {
callID := strings.TrimSpace(toolCall.ID)
if callID == "" {
callID = fmt.Sprintf("%s_call_%d", responseID, index)
}
if toolCall.Type == "" || toolCall.Type == "function" {
return dto.ResponsesOutput{
Type: responsesOutputTypeFunctionCall,
ID: callID,
Status: status,
CallId: callID,
Name: toolCall.Function.Name,
Arguments: chatArgumentsRawMessage(toolCall.Function.Arguments),
}, nil
}
return dto.ResponsesOutput{
Type: toolCall.Type,
ID: callID,
Status: status,
CallId: callID,
Arguments: toolCall.Custom,
}, nil
}
func chatArgumentsRawMessage(arguments string) []byte {
raw, err := common.Marshal(arguments)
if err != nil {
return []byte(`""`)
}
return raw
}
func chatCreatedAt(created any) int {
switch v := created.(type) {
case int:
return v
case int64:
return int(v)
case float64:
return int(v)
case float32:
return int(v)
case string:
if parsed := common.String2Int(v); parsed != 0 {
return parsed
}
}
return int(time.Now().Unix())
}
func responsesStreamEvent(eventType string, payload dto.ResponsesStreamResponse) ChatToResponsesStreamEvent {
payload.Type = eventType
return ChatToResponsesStreamEvent{
Type: eventType,
Payload: payload,
}
}
func intPtr(v int) *int {
return &v
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment