Merge remote-tracking branch 'origin/alpha' into alpha

controller/channel-test.go

@@ -200,10 +200,10 @@ func buildTestRequest(model string) *dto.GeneralOpenAIRequest {
 	} else {
 		testRequest.MaxTokens = 10
 	}
-	content, _ := json.Marshal("hi")
+
 	testMessage := dto.Message{
 		Role:    "user",
-		Content: content,
+		Content: "hi",
 	}
 	testRequest.Model = model
 	testRequest.Messages = append(testRequest.Messages, testMessage)

dto/claude.go

 package dto
 
-import "encoding/json"
+import (
+	"encoding/json"
+	"one-api/common"
+)
 
 type ClaudeMetadata struct {
 	UserId string `json:"user_id"`
@@ -23,7 +26,7 @@ type ClaudeMediaMessage struct {
 	Id        string `json:"id,omitempty"`
 	Name      string `json:"name,omitempty"`
 	Input     any    `json:"input,omitempty"`
-	Content   json.RawMessage `json:"content,omitempty"`
+	Content   any    `json:"content,omitempty"`
 	ToolUseId string `json:"tool_use_id,omitempty"`
 }
 
@@ -39,15 +42,39 @@ func (c *ClaudeMediaMessage) GetText() string {
 }
 
 func (c *ClaudeMediaMessage) IsStringContent() bool {
-	var content string
-	return json.Unmarshal(c.Content, &content) == nil
+	if c.Content == nil {
+		return false
+	}
+	_, ok := c.Content.(string)
+	if ok {
+		return true
+	}
+	return false
 }
 
 func (c *ClaudeMediaMessage) GetStringContent() string {
-	var content string
-	if err := json.Unmarshal(c.Content, &content); err == nil {
-		return content
+	if c.Content == nil {
+		return ""
+	}
+	switch c.Content.(type) {
+	case string:
+		return c.Content.(string)
+	case []any:
+		var contentStr string
+		for _, contentItem := range c.Content.([]any) {
+			contentMap, ok := contentItem.(map[string]any)
+			if !ok {
+				continue
 			}
+			if contentMap["type"] == ContentTypeText {
+				if subStr, ok := contentMap["text"].(string); ok {
+					contentStr += subStr
+				}
+			}
+		}
+		return contentStr
+	}
+
 	return ""
 }
 
@@ -57,16 +84,12 @@ func (c *ClaudeMediaMessage) GetJsonRowString() string {
 }
 
 func (c *ClaudeMediaMessage) SetContent(content any) {
-	jsonContent, _ := json.Marshal(content)
-	c.Content = jsonContent
+	c.Content = content
 }
 
 func (c *ClaudeMediaMessage) ParseMediaContent() []ClaudeMediaMessage {
-	var mediaContent []ClaudeMediaMessage
-	if err := json.Unmarshal(c.Content, &mediaContent); err == nil {
+	mediaContent, _ := common.Any2Type[[]ClaudeMediaMessage](c.Content)
 	return mediaContent
-	}
-	return make([]ClaudeMediaMessage, 0)
 }
 
 type ClaudeMessageSource struct {
@@ -82,14 +105,36 @@ type ClaudeMessage struct {
 }
 
 func (c *ClaudeMessage) IsStringContent() bool {
+	if c.Content == nil {
+		return false
+	}
 	_, ok := c.Content.(string)
 	return ok
 }
 
 func (c *ClaudeMessage) GetStringContent() string {
-	if c.IsStringContent() {
+	if c.Content == nil {
+		return ""
+	}
+	switch c.Content.(type) {
+	case string:
 		return c.Content.(string)
+	case []any:
+		var contentStr string
+		for _, contentItem := range c.Content.([]any) {
+			contentMap, ok := contentItem.(map[string]any)
+			if !ok {
+				continue
+			}
+			if contentMap["type"] == ContentTypeText {
+				if subStr, ok := contentMap["text"].(string); ok {
+					contentStr += subStr
+				}
 			}
+		}
+		return contentStr
+	}
+
 	return ""
 }
 
@@ -98,15 +143,7 @@ func (c *ClaudeMessage) SetStringContent(content string) {
 }
 
 func (c *ClaudeMessage) ParseContent() ([]ClaudeMediaMessage, error) {
-	// map content to []ClaudeMediaMessage
-	// parse to json
-	jsonContent, _ := json.Marshal(c.Content)
-	var contentList []ClaudeMediaMessage
-	err := json.Unmarshal(jsonContent, &contentList)
-	if err != nil {
-		return make([]ClaudeMediaMessage, 0), err
-	}
-	return contentList, nil
+	return common.Any2Type[[]ClaudeMediaMessage](c.Content)
 }
 
 type Tool struct {
@@ -161,14 +198,8 @@ func (c *ClaudeRequest) SetStringSystem(system string) {
 }
 
 func (c *ClaudeRequest) ParseSystem() []ClaudeMediaMessage {
-	// map content to []ClaudeMediaMessage
-	// parse to json
-	jsonContent, _ := json.Marshal(c.System)
-	var contentList []ClaudeMediaMessage
-	if err := json.Unmarshal(jsonContent, &contentList); err == nil {
-		return contentList
-	}
-	return make([]ClaudeMediaMessage, 0)
+	mediaContent, _ := common.Any2Type[[]ClaudeMediaMessage](c.System)
+	return mediaContent
 }
 
 type ClaudeError struct {

dto/openai_request.go

@@ -37,11 +37,11 @@ type GeneralOpenAIRequest struct {
 	Input               any               `json:"input,omitempty"`
 	Instruction         string            `json:"instruction,omitempty"`
 	Size                string            `json:"size,omitempty"`
-	Functions        any               `json:"functions,omitempty"`
+	Functions           json.RawMessage   `json:"functions,omitempty"`
 	FrequencyPenalty    float64           `json:"frequency_penalty,omitempty"`
 	PresencePenalty     float64           `json:"presence_penalty,omitempty"`
 	ResponseFormat      *ResponseFormat   `json:"response_format,omitempty"`
-	EncodingFormat   any               `json:"encoding_format,omitempty"`
+	EncodingFormat      json.RawMessage   `json:"encoding_format,omitempty"`
 	Seed                float64           `json:"seed,omitempty"`
 	ParallelTooCalls    *bool             `json:"parallel_tool_calls,omitempty"`
 	Tools               []ToolCallRequest `json:"tools,omitempty"`
@@ -50,10 +50,10 @@ type GeneralOpenAIRequest struct {
 	LogProbs            bool              `json:"logprobs,omitempty"`
 	TopLogProbs         int               `json:"top_logprobs,omitempty"`
 	Dimensions          int               `json:"dimensions,omitempty"`
-	Modalities       any               `json:"modalities,omitempty"`
-	Audio            any               `json:"audio,omitempty"`
+	Modalities          json.RawMessage   `json:"modalities,omitempty"`
+	Audio               json.RawMessage   `json:"audio,omitempty"`
 	EnableThinking      any               `json:"enable_thinking,omitempty"` // ali
-	ExtraBody        any               `json:"extra_body,omitempty"`
+	ExtraBody           json.RawMessage   `json:"extra_body,omitempty"`
 	WebSearchOptions    *WebSearchOptions `json:"web_search_options,omitempty"`
 	// OpenRouter Params
 	Reasoning json.RawMessage `json:"reasoning,omitempty"`
@@ -108,7 +108,7 @@ func (r *GeneralOpenAIRequest) ParseInput() []string {
 
 type Message struct {
 	Role             string          `json:"role"`
-	Content             json.RawMessage `json:"content"`
+	Content          any             `json:"content"`
 	Name             *string         `json:"name,omitempty"`
 	Prefix           *bool           `json:"prefix,omitempty"`
 	ReasoningContent string          `json:"reasoning_content,omitempty"`
@@ -116,7 +116,7 @@ type Message struct {
 	ToolCalls        json.RawMessage `json:"tool_calls,omitempty"`
 	ToolCallId       string          `json:"tool_call_id,omitempty"`
 	parsedContent    []MediaContent
-	parsedStringContent *string
+	//parsedStringContent *string
 }
 
 type MediaContent struct {
@@ -212,6 +212,180 @@ func (m *Message) SetToolCalls(toolCalls any) {
 }
 
 func (m *Message) StringContent() string {
+	switch m.Content.(type) {
+	case string:
+		return m.Content.(string)
+	case []any:
+		var contentStr string
+		for _, contentItem := range m.Content.([]any) {
+			contentMap, ok := contentItem.(map[string]any)
+			if !ok {
+				continue
+			}
+			if contentMap["type"] == ContentTypeText {
+				if subStr, ok := contentMap["text"].(string); ok {
+					contentStr += subStr
+				}
+			}
+		}
+		return contentStr
+	}
+
+	return ""
+}
+
+func (m *Message) SetNullContent() {
+	m.Content = nil
+	m.parsedContent = nil
+}
+
+func (m *Message) SetStringContent(content string) {
+	m.Content = content
+	m.parsedContent = nil
+}
+
+func (m *Message) SetMediaContent(content []MediaContent) {
+	m.Content = content
+	m.parsedContent = content
+}
+
+func (m *Message) IsStringContent() bool {
+	_, ok := m.Content.(string)
+	if ok {
+		return true
+	}
+	return false
+}
+
+func (m *Message) ParseContent() []MediaContent {
+	if m.Content == nil {
+		return nil
+	}
+	if len(m.parsedContent) > 0 {
+		return m.parsedContent
+	}
+
+	var contentList []MediaContent
+	// 先尝试解析为字符串
+	content, ok := m.Content.(string)
+	if ok {
+		contentList = []MediaContent{{
+			Type: ContentTypeText,
+			Text: content,
+		}}
+		m.parsedContent = contentList
+		return contentList
+	}
+
+	// 尝试解析为数组
+	//var arrayContent []map[string]interface{}
+
+	arrayContent, ok := m.Content.([]any)
+	if !ok {
+		return contentList
+	}
+
+	for _, contentItemAny := range arrayContent {
+		contentItem, ok := contentItemAny.(map[string]any)
+		if !ok {
+			continue
+		}
+		contentType, ok := contentItem["type"].(string)
+		if !ok {
+			continue
+		}
+
+		switch contentType {
+		case ContentTypeText:
+			if text, ok := contentItem["text"].(string); ok {
+				contentList = append(contentList, MediaContent{
+					Type: ContentTypeText,
+					Text: text,
+				})
+			}
+
+		case ContentTypeImageURL:
+			imageUrl := contentItem["image_url"]
+			temp := &MessageImageUrl{
+				Detail: "high",
+			}
+			switch v := imageUrl.(type) {
+			case string:
+				temp.Url = v
+			case map[string]interface{}:
+				url, ok1 := v["url"].(string)
+				detail, ok2 := v["detail"].(string)
+				if ok2 {
+					temp.Detail = detail
+				}
+				if ok1 {
+					temp.Url = url
+				}
+			}
+			contentList = append(contentList, MediaContent{
+				Type:     ContentTypeImageURL,
+				ImageUrl: temp,
+			})
+
+		case ContentTypeInputAudio:
+			if audioData, ok := contentItem["input_audio"].(map[string]interface{}); ok {
+				data, ok1 := audioData["data"].(string)
+				format, ok2 := audioData["format"].(string)
+				if ok1 && ok2 {
+					temp := &MessageInputAudio{
+						Data:   data,
+						Format: format,
+					}
+					contentList = append(contentList, MediaContent{
+						Type:       ContentTypeInputAudio,
+						InputAudio: temp,
+					})
+				}
+			}
+		case ContentTypeFile:
+			if fileData, ok := contentItem["file"].(map[string]interface{}); ok {
+				fileId, ok3 := fileData["file_id"].(string)
+				if ok3 {
+					contentList = append(contentList, MediaContent{
+						Type: ContentTypeFile,
+						File: &MessageFile{
+							FileId: fileId,
+						},
+					})
+				} else {
+					fileName, ok1 := fileData["filename"].(string)
+					fileDataStr, ok2 := fileData["file_data"].(string)
+					if ok1 && ok2 {
+						contentList = append(contentList, MediaContent{
+							Type: ContentTypeFile,
+							File: &MessageFile{
+								FileName: fileName,
+								FileData: fileDataStr,
+							},
+						})
+					}
+				}
+			}
+		case ContentTypeVideoUrl:
+			if videoUrl, ok := contentItem["video_url"].(string); ok {
+				contentList = append(contentList, MediaContent{
+					Type: ContentTypeVideoUrl,
+					VideoUrl: &MessageVideoUrl{
+						Url: videoUrl,
+					},
+				})
+			}
+		}
+	}
+
+	if len(contentList) > 0 {
+		m.parsedContent = contentList
+	}
+	return contentList
+}
+
+// old code
+/*func (m *Message) StringContent() string {
 	if m.parsedStringContent != nil {
 		return *m.parsedStringContent
 	}
@@ -382,7 +556,7 @@ func (m *Message) ParseContent() []MediaContent {
 		m.parsedContent = contentList
 	}
 	return contentList
-}
+}*/
 
 type WebSearchOptions struct {
 	SearchContextSize string          `json:"search_context_size,omitempty"`

main.go

@@ -25,10 +25,10 @@ import (
 	_ "net/http/pprof"
 )
 
-//go:embed web/dist
+// go:embed web/dist
 var buildFS embed.FS
 
-//go:embed web/dist/index.html
+// go:embed web/dist/index.html
 var indexPage []byte
 
 func main() {

relay/channel/ali/text.go

@@ -96,12 +96,11 @@ func embeddingResponseAli2OpenAI(response *AliEmbeddingResponse, model string) *
 }
 
 func responseAli2OpenAI(response *AliResponse) *dto.OpenAITextResponse {
-	content, _ := json.Marshal(response.Output.Text)
 	choice := dto.OpenAITextResponseChoice{
 		Index: 0,
 		Message: dto.Message{
 			Role:    "assistant",
-			Content: content,
+			Content: response.Output.Text,
 		},
 		FinishReason: response.Output.FinishReason,
 	}

relay/channel/baidu/relay-baidu.go

@@ -53,12 +53,11 @@ func requestOpenAI2Baidu(request dto.GeneralOpenAIRequest) *BaiduChatRequest {
 }
 
 func responseBaidu2OpenAI(response *BaiduChatResponse) *dto.OpenAITextResponse {
-	content, _ := json.Marshal(response.Result)
 	choice := dto.OpenAITextResponseChoice{
 		Index: 0,
 		Message: dto.Message{
 			Role:    "assistant",
-			Content: content,
+			Content: response.Result,
 		},
 		FinishReason: "stop",
 	}

relay/channel/claude/relay-claude.go

@@ -48,9 +48,9 @@ func RequestOpenAI2ClaudeComplete(textRequest dto.GeneralOpenAIRequest) *dto.Cla
 	prompt := ""
 	for _, message := range textRequest.Messages {
 		if message.Role == "user" {
-			prompt += fmt.Sprintf("\n\nHuman: %s", message.Content)
+			prompt += fmt.Sprintf("\n\nHuman: %s", message.StringContent())
 		} else if message.Role == "assistant" {
-			prompt += fmt.Sprintf("\n\nAssistant: %s", message.Content)
+			prompt += fmt.Sprintf("\n\nAssistant: %s", message.StringContent())
 		} else if message.Role == "system" {
 			if prompt == "" {
 				prompt = message.StringContent()
@@ -155,15 +155,13 @@ func RequestOpenAI2ClaudeMessage(textRequest dto.GeneralOpenAIRequest) (*dto.Cla
 		}
 		if lastMessage.Role == message.Role && lastMessage.Role != "tool" {
 			if lastMessage.IsStringContent() && message.IsStringContent() {
-				content, _ := json.Marshal(strings.Trim(fmt.Sprintf("%s %s", lastMessage.StringContent(), message.StringContent()), "\""))
-				fmtMessage.Content = content
+				fmtMessage.SetStringContent(strings.Trim(fmt.Sprintf("%s %s", lastMessage.StringContent(), message.StringContent()), "\""))
 				// delete last message
 				formatMessages = formatMessages[:len(formatMessages)-1]
 			}
 		}
 		if fmtMessage.Content == nil {
-			content, _ := json.Marshal("...")
-			fmtMessage.Content = content
+			fmtMessage.SetStringContent("...")
 		}
 		formatMessages = append(formatMessages, fmtMessage)
 		lastMessage = fmtMessage
@@ -397,12 +395,11 @@ func ResponseClaude2OpenAI(reqMode int, claudeResponse *dto.ClaudeResponse) *dto
 	thinkingContent := ""
 
 	if reqMode == RequestModeCompletion {
-		content, _ := json.Marshal(strings.TrimPrefix(claudeResponse.Completion, " "))
 		choice := dto.OpenAITextResponseChoice{
 			Index: 0,
 			Message: dto.Message{
 				Role:    "assistant",
-				Content: content,
+				Content: strings.TrimPrefix(claudeResponse.Completion, " "),
 				Name:    nil,
 			},
 			FinishReason: stopReasonClaude2OpenAI(claudeResponse.StopReason),

relay/channel/cohere/relay-cohere.go

@@ -195,11 +195,10 @@ func cohereHandler(c *gin.Context, resp *http.Response, modelName string, prompt
 	openaiResp.Model = modelName
 	openaiResp.Usage = usage
 
-	content, _ := json.Marshal(cohereResp.Text)
 	openaiResp.Choices = []dto.OpenAITextResponseChoice{
 		{
 			Index:        0,
-			Message:      dto.Message{Content: content, Role: "assistant"},
+			Message:      dto.Message{Content: cohereResp.Text, Role: "assistant"},
 			FinishReason: stopReasonCohere2OpenAI(cohereResp.FinishReason),
 		},
 	}

relay/channel/coze/dto.go

@@ -10,7 +10,7 @@ type CozeError struct {
 type CozeEnterMessage struct {
 	Role        string          `json:"role"`
 	Type        string          `json:"type,omitempty"`
-	Content     json.RawMessage `json:"content,omitempty"`
+	Content     any             `json:"content,omitempty"`
 	MetaData    json.RawMessage `json:"meta_data,omitempty"`
 	ContentType string          `json:"content_type,omitempty"`
 }

relay/channel/dify/relay-dify.go

@@ -278,12 +278,11 @@ func difyHandler(c *gin.Context, resp *http.Response, info *relaycommon.RelayInf
 		Created: common.GetTimestamp(),
 		Usage:   difyResponse.MetaData.Usage,
 	}
-	content, _ := json.Marshal(difyResponse.Answer)
 	choice := dto.OpenAITextResponseChoice{
 		Index: 0,
 		Message: dto.Message{
 			Role:    "assistant",
-			Content: content,
+			Content: difyResponse.Answer,
 		},
 		FinishReason: "stop",
 	}

relay/channel/gemini/relay-gemini.go

@@ -175,12 +175,6 @@ func CovertGemini2OpenAI(textRequest dto.GeneralOpenAIRequest, info *relaycommon
 		// common.SysLog("tools: " + fmt.Sprintf("%+v", geminiRequest.Tools))
 		// json_data, _ := json.Marshal(geminiRequest.Tools)
 		// common.SysLog("tools_json: " + string(json_data))
-	} else if textRequest.Functions != nil {
-		//geminiRequest.Tools = []GeminiChatTool{
-		//	{
-		//		FunctionDeclarations: textRequest.Functions,
-		//	},
-		//}
 	}
 
 	if textRequest.ResponseFormat != nil && (textRequest.ResponseFormat.Type == "json_schema" || textRequest.ResponseFormat.Type == "json_object") {
@@ -609,14 +603,13 @@ func responseGeminiChat2OpenAI(response *GeminiChatResponse) *dto.OpenAITextResp
 		Created: common.GetTimestamp(),
 		Choices: make([]dto.OpenAITextResponseChoice, 0, len(response.Candidates)),
 	}
-	content, _ := json.Marshal("")
 	isToolCall := false
 	for _, candidate := range response.Candidates {
 		choice := dto.OpenAITextResponseChoice{
 			Index: int(candidate.Index),
 			Message: dto.Message{
 				Role:    "assistant",
-				Content: content,
+				Content: "",
 			},
 			FinishReason: constant.FinishReasonStop,
 		}

relay/channel/mistral/text.go

@@ -47,7 +47,7 @@ func requestOpenAI2Mistral(request *dto.GeneralOpenAIRequest) *dto.GeneralOpenAI
 		}
 
 		mediaMessages := message.ParseContent()
-		if message.Role == "assistant" && message.ToolCalls != nil && string(message.Content) == "null" {
+		if message.Role == "assistant" && message.ToolCalls != nil && message.Content != "null" {
 			mediaMessages = []dto.MediaContent{}
 		}
 		for j, mediaMessage := range mediaMessages {

relay/channel/palm/relay-palm.go

@@ -45,12 +45,11 @@ func responsePaLM2OpenAI(response *PaLMChatResponse) *dto.OpenAITextResponse {
 		Choices: make([]dto.OpenAITextResponseChoice, 0, len(response.Candidates)),
 	}
 	for i, candidate := range response.Candidates {
-		content, _ := json.Marshal(candidate.Content)
 		choice := dto.OpenAITextResponseChoice{
 			Index: i,
 			Message: dto.Message{
 				Role:    "assistant",
-				Content: content,
+				Content: candidate.Content,
 			},
 			FinishReason: "stop",
 		}

relay/channel/tencent/relay-tencent.go

@@ -56,12 +56,11 @@ func responseTencent2OpenAI(response *TencentChatResponse) *dto.OpenAITextRespon
 		},
 	}
 	if len(response.Choices) > 0 {
-		content, _ := json.Marshal(response.Choices[0].Messages.Content)
 		choice := dto.OpenAITextResponseChoice{
 			Index: 0,
 			Message: dto.Message{
 				Role:    "assistant",
-				Content: content,
+				Content: response.Choices[0].Messages.Content,
 			},
 			FinishReason: response.Choices[0].FinishReason,
 		}

relay/channel/xunfei/relay-xunfei.go

@@ -61,12 +61,11 @@ func responseXunfei2OpenAI(response *XunfeiChatResponse) *dto.OpenAITextResponse
 			},
 		}
 	}
-	content, _ := json.Marshal(response.Payload.Choices.Text[0].Content)
 	choice := dto.OpenAITextResponseChoice{
 		Index: 0,
 		Message: dto.Message{
 			Role:    "assistant",
-			Content: content,
+			Content: response.Payload.Choices.Text[0].Content,
 		},
 		FinishReason: constant.FinishReasonStop,
 	}

relay/channel/zhipu/relay-zhipu.go

@@ -108,12 +108,11 @@ func responseZhipu2OpenAI(response *ZhipuResponse) *dto.OpenAITextResponse {
 		Usage:   response.Data.Usage,
 	}
 	for i, choice := range response.Data.Choices {
-		content, _ := json.Marshal(strings.Trim(choice.Content, "\""))
 		openaiChoice := dto.OpenAITextResponseChoice{
 			Index: i,
 			Message: dto.Message{
 				Role:    choice.Role,
-				Content: content,
+				Content: strings.Trim(choice.Content, "\""),
 			},
 			FinishReason: "",
 		}

service/token_counter.go

@@ -261,16 +261,20 @@ func CountTokenClaudeMessages(messages []dto.ClaudeMessage, model string, stream
 					//}
 					tokenNum += 1000
 				case "tool_use":
+					if mediaMessage.Input != nil {
 						tokenNum += getTokenNum(tokenEncoder, mediaMessage.Name)
 						inputJSON, _ := json.Marshal(mediaMessage.Input)
 						tokenNum += getTokenNum(tokenEncoder, string(inputJSON))
+					}
 				case "tool_result":
+					if mediaMessage.Content != nil {
 						contentJSON, _ := json.Marshal(mediaMessage.Content)
 						tokenNum += getTokenNum(tokenEncoder, string(contentJSON))
 					}
 				}
 			}
 		}
+	}
 
 	// Add a constant for message formatting (this may need adjustment based on Claude's exact formatting)
 	tokenNum += len(messages) * 2 // Assuming 2 tokens per message for formatting
@@ -386,7 +390,7 @@ func CountTokenMessages(info *relaycommon.RelayInfo, messages []dto.Message, mod
 	for _, message := range messages {
 		tokenNum += tokensPerMessage
 		tokenNum += getTokenNum(tokenEncoder, message.Role)
-		if len(message.Content) > 0 {
+		if message.Content != nil {
 			if message.Name != nil {
 				tokenNum += tokensPerName
 				tokenNum += getTokenNum(tokenEncoder, *message.Name)