feat: doubao tts support streaming realtime audio

relay/channel/volcengine/adaptor.go

@@ -70,7 +70,7 @@ func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInf
 		Request: VolcengineTTSReqInfo{
 		Request: VolcengineTTSReqInfo{
 			ReqID:     generateRequestID(),
 			ReqID:     generateRequestID(),
 			Text:      request.Input,
 			Text:      request.Input,
-			Operation: "query",
+			Operation: "submit", // WebSocket uses "submit"
 			Model:     info.OriginModelName,
 			Model:     info.OriginModelName,
 		},
 		},
 	}
 	}
@@ -82,12 +82,11 @@ func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInf
 		}
 		}
 	}
 	}
 
 
-	jsonData, err := json.Marshal(volcRequest)
-	if err != nil {
-		return nil, fmt.Errorf("error marshalling volcengine request: %w", err)
-	}
+	// Store the request in context for WebSocket handler
+	c.Set("volcengine_tts_request", volcRequest)
 
 
-	return bytes.NewReader(jsonData), nil
+	// Return nil as WebSocket doesn't use traditional request body
+	return nil, nil
 }
 }
 
 
 func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (any, error) {
 func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (any, error) {
@@ -268,7 +267,7 @@ func (a *Adaptor) GetRequestURL(info *relaycommon.RelayInfo) (string, error) {
 		case constant.RelayModeAudioSpeech:
 		case constant.RelayModeAudioSpeech:
 			// 只有当 baseUrl 是火山默认的官方Url时才改为官方的的TTS接口，否则走透传的New接口
 			// 只有当 baseUrl 是火山默认的官方Url时才改为官方的的TTS接口，否则走透传的New接口
 			if baseUrl == channelconstant.ChannelBaseURLs[channelconstant.ChannelTypeVolcEngine] {
 			if baseUrl == channelconstant.ChannelBaseURLs[channelconstant.ChannelTypeVolcEngine] {
-				return "https://openspeech.bytedance.com/api/v1/tts", nil
+				return "wss://openspeech.bytedance.com/api/v1/tts/ws_binary", nil
 			}
 			}
 			return fmt.Sprintf("%s/v1/audio/speech", baseUrl), nil
 			return fmt.Sprintf("%s/v1/audio/speech", baseUrl), nil
 		default:
 		default:
@@ -320,12 +319,60 @@ func (a *Adaptor) ConvertOpenAIResponsesRequest(c *gin.Context, info *relaycommo
 }
 }
 
 
 func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (any, error) {
 func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (any, error) {
+	// For TTS with WebSocket, skip traditional HTTP request
+	if info.RelayMode == constant.RelayModeAudioSpeech {
+		baseUrl := info.ChannelBaseUrl
+		if baseUrl == "" {
+			baseUrl = channelconstant.ChannelBaseURLs[channelconstant.ChannelTypeVolcEngine]
+		}
+		// Only use WebSocket for official Volcengine endpoint
+		if baseUrl == channelconstant.ChannelBaseURLs[channelconstant.ChannelTypeVolcEngine] {
+			return nil, nil // WebSocket handling will be done in DoResponse
+		}
+	}
 	return channel.DoApiRequest(a, c, info, requestBody)
 	return channel.DoApiRequest(a, c, info, requestBody)
 }
 }
 
 
 func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (usage any, err *types.NewAPIError) {
 func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (usage any, err *types.NewAPIError) {
 	if info.RelayMode == constant.RelayModeAudioSpeech {
 	if info.RelayMode == constant.RelayModeAudioSpeech {
 		encoding := mapEncoding(c.GetString("response_format"))
 		encoding := mapEncoding(c.GetString("response_format"))
+
+		// Check if this is WebSocket mode (resp will be nil for WebSocket)
+		if resp == nil {
+			// Get the WebSocket URL
+			requestURL, urlErr := a.GetRequestURL(info)
+			if urlErr != nil {
+				return nil, types.NewErrorWithStatusCode(
+					urlErr,
+					types.ErrorCodeBadRequestBody,
+					http.StatusInternalServerError,
+				)
+			}
+
+			// Retrieve the volcengine request from context
+			volcRequestInterface, exists := c.Get("volcengine_tts_request")
+			if !exists {
+				return nil, types.NewErrorWithStatusCode(
+					errors.New("volcengine TTS request not found in context"),
+					types.ErrorCodeBadRequestBody,
+					http.StatusInternalServerError,
+				)
+			}
+
+			volcRequest, ok := volcRequestInterface.(VolcengineTTSRequest)
+			if !ok {
+				return nil, types.NewErrorWithStatusCode(
+					errors.New("invalid volcengine TTS request type"),
+					types.ErrorCodeBadRequestBody,
+					http.StatusInternalServerError,
+				)
+			}
+
+			// Handle WebSocket streaming
+			return handleTTSWebSocketResponse(c, requestURL, volcRequest, info, encoding)
+		}
+
+		// Handle traditional HTTP response
 		return handleTTSResponse(c, resp, info, encoding)
 		return handleTTSResponse(c, resp, info, encoding)
 	}
 	}
 
 

relay/channel/volcengine/tts.go

 package volcengine
 package volcengine
 
 
 import (
 import (
+	"context"
 	"encoding/base64"
 	"encoding/base64"
 	"encoding/json"
 	"encoding/json"
 	"errors"
 	"errors"
+	"fmt"
 	"io"
 	"io"
 	"net/http"
 	"net/http"
 	"strings"
 	"strings"
@@ -13,6 +15,7 @@ import (
 	"github.com/QuantumNous/new-api/types"
 	"github.com/QuantumNous/new-api/types"
 	"github.com/gin-gonic/gin"
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
 	"github.com/google/uuid"
+	"github.com/gorilla/websocket"
 )
 )
 
 
 type VolcengineTTSRequest struct {
 type VolcengineTTSRequest struct {
@@ -192,3 +195,129 @@ func handleTTSResponse(c *gin.Context, resp *http.Response, info *relaycommon.Re
 func generateRequestID() string {
 func generateRequestID() string {
 	return uuid.New().String()
 	return uuid.New().String()
 }
 }
+
+// handleTTSWebSocketResponse handles streaming TTS response via WebSocket
+func handleTTSWebSocketResponse(c *gin.Context, requestURL string, volcRequest VolcengineTTSRequest, info *relaycommon.RelayInfo, encoding string) (usage any, err *types.NewAPIError) {
+	// Parse API key for auth
+	_, token, parseErr := parseVolcengineAuth(info.ApiKey)
+	if parseErr != nil {
+		return nil, types.NewErrorWithStatusCode(
+			parseErr,
+			types.ErrorCodeChannelInvalidKey,
+			http.StatusUnauthorized,
+		)
+	}
+
+	// Setup WebSocket headers
+	header := http.Header{}
+	header.Set("Authorization", fmt.Sprintf("Bearer;%s", token))
+
+	// Dial WebSocket connection
+	conn, resp, dialErr := websocket.DefaultDialer.DialContext(context.Background(), requestURL, header)
+	if dialErr != nil {
+		if resp != nil {
+			return nil, types.NewErrorWithStatusCode(
+				fmt.Errorf("failed to connect to websocket: %w, status: %d", dialErr, resp.StatusCode),
+				types.ErrorCodeBadResponseStatusCode,
+				http.StatusBadGateway,
+			)
+		}
+		return nil, types.NewErrorWithStatusCode(
+			fmt.Errorf("failed to connect to websocket: %w", dialErr),
+			types.ErrorCodeBadResponseStatusCode,
+			http.StatusBadGateway,
+		)
+	}
+	defer conn.Close()
+
+	// Update request operation to "submit" for WebSocket
+	volcRequest.Request.Operation = "submit"
+
+	// Marshal request payload
+	payload, marshalErr := json.Marshal(volcRequest)
+	if marshalErr != nil {
+		return nil, types.NewErrorWithStatusCode(
+			fmt.Errorf("failed to marshal request: %w", marshalErr),
+			types.ErrorCodeBadRequestBody,
+			http.StatusInternalServerError,
+		)
+	}
+
+	// Send full client request
+	if sendErr := FullClientRequest(conn, payload); sendErr != nil {
+		return nil, types.NewErrorWithStatusCode(
+			fmt.Errorf("failed to send request: %w", sendErr),
+			types.ErrorCodeBadRequestBody,
+			http.StatusInternalServerError,
+		)
+	}
+
+	// Set response headers
+	contentType := getContentTypeByEncoding(encoding)
+	c.Header("Content-Type", contentType)
+	c.Header("Transfer-Encoding", "chunked")
+
+	// Stream audio data
+	var audioBuffer []byte
+	for {
+		msg, recvErr := ReceiveMessage(conn)
+		if recvErr != nil {
+			if websocket.IsCloseError(recvErr, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
+				break
+			}
+			return nil, types.NewErrorWithStatusCode(
+				fmt.Errorf("failed to receive message: %w", recvErr),
+				types.ErrorCodeBadResponse,
+				http.StatusInternalServerError,
+			)
+		}
+
+		switch msg.MsgType {
+		case MsgTypeError:
+			return nil, types.NewErrorWithStatusCode(
+				fmt.Errorf("received error from server: code=%d, %s", msg.ErrorCode, string(msg.Payload)),
+				types.ErrorCodeBadResponse,
+				http.StatusBadRequest,
+			)
+		case MsgTypeFrontEndResultServer:
+			// Metadata response, can be logged or processed
+			continue
+		case MsgTypeAudioOnlyServer:
+			// Stream audio chunk to client
+			if len(msg.Payload) > 0 {
+				audioBuffer = append(audioBuffer, msg.Payload...)
+				if _, writeErr := c.Writer.Write(msg.Payload); writeErr != nil {
+					return nil, types.NewErrorWithStatusCode(
+						fmt.Errorf("failed to write audio data: %w", writeErr),
+						types.ErrorCodeBadResponse,
+						http.StatusInternalServerError,
+					)
+				}
+				c.Writer.Flush()
+			}
+
+			// Check if this is the last packet (negative sequence)
+			if msg.Sequence < 0 {
+				c.Status(http.StatusOK)
+				usage = &dto.Usage{
+					PromptTokens:     info.PromptTokens,
+					CompletionTokens: 0,
+					TotalTokens:      info.PromptTokens,
+				}
+				return usage, nil
+			}
+		default:
+			// Unknown message type, log and continue
+			continue
+		}
+	}
+
+	// If we reach here, connection closed without final packet
+	c.Status(http.StatusOK)
+	usage = &dto.Usage{
+		PromptTokens:     info.PromptTokens,
+		CompletionTokens: 0,
+		TotalTokens:      info.PromptTokens,
+	}
+	return usage, nil
+}