Skip to content

phsl / new-api

Go to a project
Commit 8ad159a3 by Seefs Committed by GitHub
Options

fix(ollama): preserve reasoning and tool-call context (#6605)

parent 85feb7a3
Showing with 108 additions and 46 deletions
relay/channel/ollama/dto.go
...@@ -10,6 +10,7 @@ type OllamaChatMessage struct { ...@@ -10,6 +10,7 @@ type OllamaChatMessage struct {
Images []string `json:"images,omitempty"` Images []string `json:"images,omitempty"`
ToolCalls []OllamaToolCall `json:"tool_calls,omitempty"` ToolCalls []OllamaToolCall `json:"tool_calls,omitempty"`
ToolName string `json:"tool_name,omitempty"` ToolName string `json:"tool_name,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
Thinking json.RawMessage `json:"thinking,omitempty"` Thinking json.RawMessage `json:"thinking,omitempty"`
} }
...@@ -25,6 +26,7 @@ type OllamaTool struct { ...@@ -25,6 +26,7 @@ type OllamaTool struct {
} }
type OllamaToolCall struct { type OllamaToolCall struct {
ID string `json:"id,omitempty"`
Function struct { Function struct {
Name string `json:"name"` Name string `json:"name"`
Arguments interface{} `json:"arguments"` Arguments interface{} `json:"arguments"`
...@@ -36,7 +38,7 @@ type OllamaChatRequest struct { ...@@ -36,7 +38,7 @@ type OllamaChatRequest struct {
Messages []OllamaChatMessage `json:"messages"` Messages []OllamaChatMessage `json:"messages"`
Tools interface{} `json:"tools,omitempty"` Tools interface{} `json:"tools,omitempty"`
Format interface{} `json:"format,omitempty"` Format interface{} `json:"format,omitempty"`
Stream bool `json:"stream,omitempty"` Stream bool `json:"stream"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
KeepAlive interface{} `json:"keep_alive,omitempty"` KeepAlive interface{} `json:"keep_alive,omitempty"`
Think json.RawMessage `json:"think,omitempty"` Think json.RawMessage `json:"think,omitempty"`
...@@ -48,7 +50,7 @@ type OllamaGenerateRequest struct { ...@@ -48,7 +50,7 @@ type OllamaGenerateRequest struct {
Suffix string `json:"suffix,omitempty"` Suffix string `json:"suffix,omitempty"`
Images []string `json:"images,omitempty"` Images []string `json:"images,omitempty"`
Format interface{} `json:"format,omitempty"` Format interface{} `json:"format,omitempty"`
Stream bool `json:"stream,omitempty"` Stream bool `json:"stream"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
KeepAlive interface{} `json:"keep_alive,omitempty"` KeepAlive interface{} `json:"keep_alive,omitempty"`
Think json.RawMessage `json:"think,omitempty"` Think json.RawMessage `json:"think,omitempty"`
......
relay/channel/ollama/relay-ollama.go
package ollama package ollama
import ( import (
"encoding/json"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
...@@ -19,24 +18,67 @@ import ( ...@@ -19,24 +18,67 @@ import (
"github.com/samber/lo" "github.com/samber/lo"
) )
func toOllamaResponseFormat(responseFormat *dto.ResponseFormat) (any, error) {
if responseFormat == nil {
return nil, nil
}
switch responseFormat.Type {
case "json", "json_object":
return "json", nil
case "json_schema":
if len(responseFormat.JsonSchema) == 0 {
return nil, nil
}
var jsonSchema dto.FormatJsonSchema
if err := common.Unmarshal(responseFormat.JsonSchema, &jsonSchema); err != nil {
return nil, fmt.Errorf("invalid ollama response format: %w", err)
}
return jsonSchema.Schema, nil
default:
return nil, nil
}
}
func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*OllamaChatRequest, error) { func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*OllamaChatRequest, error) {
think := r.Think
if len(think) == 0 {
effort := r.ReasoningEffort
if len(r.Reasoning) > 0 {
var reasoning dto.Reasoning
if err := common.Unmarshal(r.Reasoning, &reasoning); err != nil {
return nil, fmt.Errorf("invalid ollama reasoning: %w", err)
}
effort = lo.CoalesceOrEmpty(reasoning.Effort, effort)
}
if effort != "" {
var thinkValue any
switch effort {
case "none":
thinkValue = false
case "low", "medium", "high", "max":
thinkValue = effort
default:
return nil, fmt.Errorf("unsupported ollama reasoning effort %q", effort)
}
var err error
think, err = common.Marshal(thinkValue)
if err != nil {
return nil, fmt.Errorf("marshal ollama think: %w", err)
}
}
}
chatReq := &OllamaChatRequest{ chatReq := &OllamaChatRequest{
Model: r.Model, Model: r.Model,
Stream: lo.FromPtrOr(r.Stream, false), Stream: lo.FromPtrOr(r.Stream, false),
Options: map[string]any{}, Options: map[string]any{},
Think: r.Think, Think: think,
}
if r.ResponseFormat != nil {
if r.ResponseFormat.Type == "json" {
chatReq.Format = "json"
} else if r.ResponseFormat.Type == "json_schema" {
if len(r.ResponseFormat.JsonSchema) > 0 {
var schema any
_ = json.Unmarshal(r.ResponseFormat.JsonSchema, &schema)
chatReq.Format = schema
}
} }
format, err := toOllamaResponseFormat(r.ResponseFormat)
if err != nil {
return nil, err
} }
chatReq.Format = format
// options mapping // options mapping
if r.Temperature != nil { if r.Temperature != nil {
...@@ -68,12 +110,10 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam ...@@ -68,12 +110,10 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam
case []string: case []string:
chatReq.Options["stop"] = v chatReq.Options["stop"] = v
case []any: case []any:
arr := make([]string, 0, len(v)) arr := lo.FilterMap(v, func(item any, _ int) (string, bool) {
for _, i := range v { value, ok := item.(string)
if s, ok := i.(string); ok { return value, ok
arr = append(arr, s) })
}
}
if len(arr) > 0 { if len(arr) > 0 {
chatReq.Options["stop"] = arr chatReq.Options["stop"] = arr
} }
...@@ -81,14 +121,20 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam ...@@ -81,14 +121,20 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam
} }
if len(r.Tools) > 0 { if len(r.Tools) > 0 {
tools := make([]OllamaTool, 0, len(r.Tools)) chatReq.Tools = lo.Map(r.Tools, func(tool dto.ToolCallRequest, _ int) OllamaTool {
for _, t := range r.Tools { return OllamaTool{
tools = append(tools, OllamaTool{Type: "function", Function: OllamaToolFunction{Name: t.Function.Name, Description: t.Function.Description, Parameters: t.Function.Parameters}}) Type: "function",
Function: OllamaToolFunction{
Name: tool.Function.Name,
Description: tool.Function.Description,
Parameters: tool.Function.Parameters,
},
} }
chatReq.Tools = tools })
} }
chatReq.Messages = make([]OllamaChatMessage, 0, len(r.Messages)) chatReq.Messages = make([]OllamaChatMessage, 0, len(r.Messages))
toolNamesByCallID := make(map[string]string)
for _, m := range r.Messages { for _, m := range r.Messages {
var textBuilder strings.Builder var textBuilder strings.Builder
var images []string var images []string
...@@ -117,8 +163,18 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam ...@@ -117,8 +163,18 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam
if len(images) > 0 { if len(images) > 0 {
cm.Images = images cm.Images = images
} }
if m.Role == "tool" && m.Name != nil { if m.Role == "assistant" {
cm.ToolName = *m.Name if reasoning, ok := lo.Coalesce(m.ReasoningContent, m.Reasoning); ok {
thinking, err := common.Marshal(*reasoning)
if err != nil {
return nil, fmt.Errorf("marshal ollama thinking: %w", err)
}
cm.Thinking = thinking
}
}
if m.Role == "tool" {
cm.ToolCallID = m.ToolCallId
cm.ToolName = lo.CoalesceOrEmpty(lo.FromPtr(m.Name), toolNamesByCallID[m.ToolCallId])
} }
if m.ToolCalls != nil && len(m.ToolCalls) > 0 { if m.ToolCalls != nil && len(m.ToolCalls) > 0 {
parsed := m.ParseToolCalls() parsed := m.ParseToolCalls()
...@@ -127,15 +183,18 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam ...@@ -127,15 +183,18 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam
for _, tc := range parsed { for _, tc := range parsed {
var args interface{} var args interface{}
if tc.Function.Arguments != "" { if tc.Function.Arguments != "" {
_ = json.Unmarshal([]byte(tc.Function.Arguments), &args) _ = common.Unmarshal([]byte(tc.Function.Arguments), &args)
} }
if args == nil { if args == nil {
args = map[string]any{} args = map[string]any{}
} }
oc := OllamaToolCall{} oc := OllamaToolCall{ID: tc.ID}
oc.Function.Name = tc.Function.Name oc.Function.Name = tc.Function.Name
oc.Function.Arguments = args oc.Function.Arguments = args
calls = append(calls, oc) calls = append(calls, oc)
if tc.ID != "" {
toolNamesByCallID[tc.ID] = tc.Function.Name
}
} }
cm.ToolCalls = calls cm.ToolCalls = calls
} }
...@@ -175,15 +234,11 @@ func openAIToGenerate(c *gin.Context, r *dto.GeneralOpenAIRequest) (*OllamaGener ...@@ -175,15 +234,11 @@ func openAIToGenerate(c *gin.Context, r *dto.GeneralOpenAIRequest) (*OllamaGener
gen.Suffix = s gen.Suffix = s
} }
} }
if r.ResponseFormat != nil { format, err := toOllamaResponseFormat(r.ResponseFormat)
if r.ResponseFormat.Type == "json" { if err != nil {
gen.Format = "json" return nil, err
} else if r.ResponseFormat.Type == "json_schema" {
var schema any
_ = json.Unmarshal(r.ResponseFormat.JsonSchema, &schema)
gen.Format = schema
}
} }
gen.Format = format
if r.Temperature != nil { if r.Temperature != nil {
gen.Options["temperature"] = r.Temperature gen.Options["temperature"] = r.Temperature
} }
...@@ -212,12 +267,10 @@ func openAIToGenerate(c *gin.Context, r *dto.GeneralOpenAIRequest) (*OllamaGener ...@@ -212,12 +267,10 @@ func openAIToGenerate(c *gin.Context, r *dto.GeneralOpenAIRequest) (*OllamaGener
case []string: case []string:
gen.Options["stop"] = v gen.Options["stop"] = v
case []any: case []any:
arr := make([]string, 0, len(v)) arr := lo.FilterMap(v, func(item any, _ int) (string, bool) {
for _, i := range v { value, ok := item.(string)
if s, ok := i.(string); ok { return value, ok
arr = append(arr, s) })
}
}
if len(arr) > 0 { if len(arr) > 0 {
gen.Options["stop"] = arr gen.Options["stop"] = arr
} }
...@@ -510,7 +563,7 @@ func FetchOllamaVersion(baseURL, apiKey string) (string, error) { ...@@ -510,7 +563,7 @@ func FetchOllamaVersion(baseURL, apiKey string) (string, error) {
Version string `json:"version"` Version string `json:"version"`
} }
if err := json.Unmarshal(body, &versionResp); err != nil { if err := common.Unmarshal(body, &versionResp); err != nil {
return "", fmt.Errorf("解析响应失败: %v", err) return "", fmt.Errorf("解析响应失败: %v", err)
} }
......
relay/channel/ollama/stream.go
...@@ -58,8 +58,12 @@ func ollamaToolCallsToOpenAI(toolCalls []OllamaToolCall, startIndex int, include ...@@ -58,8 +58,12 @@ func ollamaToolCallsToOpenAI(toolCalls []OllamaToolCall, startIndex int, include
argBytes = []byte("{}") argBytes = []byte("{}")
} }
} }
toolCallID := tc.ID
if toolCallID == "" {
toolCallID = fmt.Sprintf("call_%d", startIndex)
}
tr := dto.ToolCallResponse{ tr := dto.ToolCallResponse{
ID: fmt.Sprintf("call_%d", startIndex), ID: toolCallID,
Type: "function", Type: "function",
Function: dto.FunctionResponse{ Function: dto.FunctionResponse{
Name: tc.Function.Name, Name: tc.Function.Name,
......
relay/channel/ollama/stream_test.go
...@@ -23,10 +23,12 @@ func TestOllamaChatHandlerNonStreamToolCalls(t *testing.T) { ...@@ -23,10 +23,12 @@ func TestOllamaChatHandlerNonStreamToolCalls(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
raw string raw string
wantID string
}{ }{
{ {
name: "compact json per-line parse path", name: "compact json per-line parse path",
raw: `{"model":"llama3.1","created_at":"2026-05-27T12:00:00Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"get_weather","arguments":{"city":"Paris","days":0}}}]},"done":true,"done_reason":"stop","prompt_eval_count":5,"eval_count":7}`, raw: `{"model":"llama3.1","created_at":"2026-05-27T12:00:00Z","message":{"role":"assistant","content":"","tool_calls":[{"id":"call_upstream","function":{"name":"get_weather","arguments":{"city":"Paris","days":0}}}]},"done":true,"done_reason":"stop","prompt_eval_count":5,"eval_count":7}`,
wantID: "call_upstream",
}, },
{ {
name: "pretty json fallback parse path", name: "pretty json fallback parse path",
...@@ -53,6 +55,7 @@ func TestOllamaChatHandlerNonStreamToolCalls(t *testing.T) { ...@@ -53,6 +55,7 @@ func TestOllamaChatHandlerNonStreamToolCalls(t *testing.T) {
"prompt_eval_count": 5, "prompt_eval_count": 5,
"eval_count": 7 "eval_count": 7
}`, }`,
wantID: "call_0",
}, },
} }
...@@ -82,7 +85,7 @@ func TestOllamaChatHandlerNonStreamToolCalls(t *testing.T) { ...@@ -82,7 +85,7 @@ func TestOllamaChatHandlerNonStreamToolCalls(t *testing.T) {
var toolCalls []dto.ToolCallResponse var toolCalls []dto.ToolCallResponse
require.NoError(t, common.Unmarshal(out.Choices[0].Message.ToolCalls, &toolCalls)) require.NoError(t, common.Unmarshal(out.Choices[0].Message.ToolCalls, &toolCalls))
require.Len(t, toolCalls, 1) require.Len(t, toolCalls, 1)
assert.NotEmpty(t, toolCalls[0].ID) assert.Equal(t, tt.wantID, toolCalls[0].ID)
assert.Equal(t, "function", toolCalls[0].Type) assert.Equal(t, "function", toolCalls[0].Type)
assert.Equal(t, "get_weather", toolCalls[0].Function.Name) assert.Equal(t, "get_weather", toolCalls[0].Function.Name)
assert.Nil(t, toolCalls[0].Index) assert.Nil(t, toolCalls[0].Index)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment