Skip to content

phsl / new-api

Go to a project
Commit 52858ad1 by Scott Committed by GitHub
Options

feat: support Wan2.7 i2v media mapping (#4984) 

* feat: support Wan2.7 i2v media mapping

* fix: normalize wan2.7 i2v image inputs
parent fda81778
Showing with 323 additions and 14 deletions
relay/channel/task/ali/adaptor.go
...@@ -33,15 +33,22 @@ type AliVideoRequest struct { ...@@ -33,15 +33,22 @@ type AliVideoRequest struct {
Parameters *AliVideoParameters `json:"parameters,omitempty"` Parameters *AliVideoParameters `json:"parameters,omitempty"`
} }
// AliVideoMedia describes Wan2.7 image-to-video media inputs.
type AliVideoMedia struct {
Type string `json:"type"`
URL string `json:"url"`
}
// AliVideoInput 视频输入参数 // AliVideoInput 视频输入参数
type AliVideoInput struct { type AliVideoInput struct {
Prompt string `json:"prompt,omitempty"` // 文本提示词 Prompt string `json:"prompt,omitempty"` // 文本提示词
ImgURL string `json:"img_url,omitempty"` // 首帧图像URL或Base64（图生视频） ImgURL string `json:"img_url,omitempty"` // 首帧图像URL或Base64（图生视频）
FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频） FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频）
LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频） LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频）
AudioURL string `json:"audio_url,omitempty"` // 音频URL（wan2.5支持） AudioURL string `json:"audio_url,omitempty"` // 音频URL（wan2.5支持）
NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词 Media []AliVideoMedia `json:"media,omitempty"` // 媒体列表（wan2.7-i2v新协议）
Template string `json:"template,omitempty"` // 视频特效模板 NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词
Template string `json:"template,omitempty"` // 视频特效模板
} }
// AliVideoParameters 视频参数 // AliVideoParameters 视频参数
...@@ -87,12 +94,13 @@ type AliUsage struct { ...@@ -87,12 +94,13 @@ type AliUsage struct {
type AliMetadata struct { type AliMetadata struct {
// Input 相关 // Input 相关
AudioURL string `json:"audio_url,omitempty"` // 音频URL AudioURL string `json:"audio_url,omitempty"` // 音频URL
ImgURL string `json:"img_url,omitempty"` // 图片URL（图生视频） ImgURL string `json:"img_url,omitempty"` // 图片URL（图生视频）
FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频） FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频）
LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频） LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频）
NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词 Media []AliVideoMedia `json:"media,omitempty"` // 媒体列表（wan2.7-i2v新协议）
Template string `json:"template,omitempty"` // 视频特效模板 NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词
Template string `json:"template,omitempty"` // 视频特效模板
// Parameters 相关 // Parameters 相关
Resolution *string `json:"resolution,omitempty"` // 分辨率: 480P/720P/1080P Resolution *string `json:"resolution,omitempty"` // 分辨率: 480P/720P/1080P
...@@ -252,6 +260,93 @@ func ProcessAliOtherRatios(aliReq *AliVideoRequest) (map[string]float64, error) ...@@ -252,6 +260,93 @@ func ProcessAliOtherRatios(aliReq *AliVideoRequest) (map[string]float64, error)
return otherRatios, nil return otherRatios, nil
} }
func isWan27I2VModel(model string) bool {
return strings.HasPrefix(model, "wan2.7-i2v")
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
trimmed := strings.TrimSpace(value)
if trimmed != "" {
return trimmed
}
}
return ""
}
func firstTaskImage(req relaycommon.TaskSubmitReq) string {
if image := strings.TrimSpace(req.Image); image != "" {
return image
}
for _, image := range req.Images {
if trimmed := strings.TrimSpace(image); trimmed != "" {
return trimmed
}
}
if inputReference := strings.TrimSpace(req.InputReference); inputReference != "" {
return inputReference
}
return ""
}
func secondTaskImage(req relaycommon.TaskSubmitReq) string {
nonEmptyImages := 0
for _, image := range req.Images {
trimmed := strings.TrimSpace(image)
if trimmed == "" {
continue
}
nonEmptyImages++
if nonEmptyImages == 2 {
return trimmed
}
}
return ""
}
func normalizeWan27I2VInput(aliReq *AliVideoRequest, req relaycommon.TaskSubmitReq) error {
if !isWan27I2VModel(aliReq.Model) {
return nil
}
if len(aliReq.Input.Media) == 0 {
firstFrameURL := firstNonEmpty(aliReq.Input.FirstFrameURL, aliReq.Input.ImgURL, firstTaskImage(req))
lastFrameURL := firstNonEmpty(aliReq.Input.LastFrameURL, secondTaskImage(req))
audioURL := aliReq.Input.AudioURL
if firstFrameURL != "" {
aliReq.Input.Media = append(aliReq.Input.Media, AliVideoMedia{
Type: "first_frame",
URL: firstFrameURL,
})
}
if lastFrameURL != "" {
aliReq.Input.Media = append(aliReq.Input.Media, AliVideoMedia{
Type: "last_frame",
URL: lastFrameURL,
})
}
if audioURL != "" {
aliReq.Input.Media = append(aliReq.Input.Media, AliVideoMedia{
Type: "driving_audio",
URL: audioURL,
})
}
}
if len(aliReq.Input.Media) == 0 {
return fmt.Errorf("wan2.7-i2v requires image, images, input_reference, or input.media")
}
// Wan2.7 image-to-video uses the new input.media protocol. Avoid sending
// legacy fields that belong to wan2.6 and earlier image-to-video APIs.
aliReq.Input.ImgURL = ""
aliReq.Input.FirstFrameURL = ""
aliReq.Input.LastFrameURL = ""
aliReq.Input.AudioURL = ""
return nil
}
func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relaycommon.TaskSubmitReq) (*AliVideoRequest, error) { func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relaycommon.TaskSubmitReq) (*AliVideoRequest, error) {
upstreamModel := req.Model upstreamModel := req.Model
if info.IsModelMapped { if info.IsModelMapped {
...@@ -261,7 +356,7 @@ func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relay ...@@ -261,7 +356,7 @@ func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relay
Model: upstreamModel, Model: upstreamModel,
Input: AliVideoInput{ Input: AliVideoInput{
Prompt: req.Prompt, Prompt: req.Prompt,
ImgURL: req.InputReference, ImgURL: firstTaskImage(req),
}, },
Parameters: &AliVideoParameters{ Parameters: &AliVideoParameters{
PromptExtend: true, // 默认开启智能改写 PromptExtend: true, // 默认开启智能改写
...@@ -340,6 +435,10 @@ func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relay ...@@ -340,6 +435,10 @@ func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relay
return nil, errors.New("can't change model with metadata") return nil, errors.New("can't change model with metadata")
} }
if err := normalizeWan27I2VInput(aliReq, req); err != nil {
return nil, err
}
return aliReq, nil return aliReq, nil
} }
......
relay/channel/task/ali/adaptor_test.go 0 → 100644
package ali
import (
"strings"
"testing"
"github.com/QuantumNous/new-api/common"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/stretchr/testify/require"
)
func testRelayInfo() *relaycommon.RelayInfo {
return &relaycommon.RelayInfo{
ChannelMeta: &relaycommon.ChannelMeta{},
}
}
func TestConvertToAliRequestWan27I2VBuildsMediaFromImage(t *testing.T) {
adaptor := &TaskAdaptor{}
req := relaycommon.TaskSubmitReq{
Model: "wan2.7-i2v",
Prompt: "animate the first frame",
Image: "https://example.com/first.png",
Size: "720p",
Duration: 10,
}
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
require.NoError(t, err)
require.Equal(t, "wan2.7-i2v", aliReq.Model)
require.Equal(t, "720P", aliReq.Parameters.Resolution)
require.Equal(t, 10, aliReq.Parameters.Duration)
require.Equal(t, []AliVideoMedia{
{Type: "first_frame", URL: "https://example.com/first.png"},
}, aliReq.Input.Media)
require.Empty(t, aliReq.Input.ImgURL)
body, err := common.Marshal(aliReq)
require.NoError(t, err)
require.Contains(t, string(body), `"media"`)
require.NotContains(t, string(body), `"img_url"`)
}
func TestConvertToAliRequestWan27I2VBuildsFirstAndLastFrameFromImages(t *testing.T) {
adaptor := &TaskAdaptor{}
req := relaycommon.TaskSubmitReq{
Model: "wan2.7-i2v",
Prompt: "interpolate between frames",
Images: []string{
"https://example.com/first.png",
"https://example.com/last.png",
},
}
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
require.NoError(t, err)
require.Equal(t, []AliVideoMedia{
{Type: "first_frame", URL: "https://example.com/first.png"},
{Type: "last_frame", URL: "https://example.com/last.png"},
}, aliReq.Input.Media)
}
func TestConvertToAliRequestWan27I2VPrefersImageBeforeImagesAndInputReference(t *testing.T) {
adaptor := &TaskAdaptor{}
req := relaycommon.TaskSubmitReq{
Model: "wan2.7-i2v",
Prompt: "use the direct image",
Image: " https://example.com/direct.png ",
Images: []string{"https://example.com/images-first.png", " https://example.com/images-last.png "},
InputReference: "https://example.com/input-reference.png",
}
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
require.NoError(t, err)
require.Equal(t, []AliVideoMedia{
{Type: "first_frame", URL: "https://example.com/direct.png"},
{Type: "last_frame", URL: "https://example.com/images-last.png"},
}, aliReq.Input.Media)
}
func TestConvertToAliRequestWan27I2VFallsBackToFirstNonEmptyImage(t *testing.T) {
adaptor := &TaskAdaptor{}
req := relaycommon.TaskSubmitReq{
Model: "wan2.7-i2v",
Prompt: "skip blank images",
Image: " ",
Images: []string{
" ",
" https://example.com/first.png ",
" https://example.com/last.png ",
},
InputReference: "https://example.com/input-reference.png",
}
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
require.NoError(t, err)
require.Equal(t, []AliVideoMedia{
{Type: "first_frame", URL: "https://example.com/first.png"},
{Type: "last_frame", URL: "https://example.com/last.png"},
}, aliReq.Input.Media)
}
func TestConvertToAliRequestWan27I2VKeepsExplicitMetadataMedia(t *testing.T) {
adaptor := &TaskAdaptor{}
req := relaycommon.TaskSubmitReq{
Model: "wan2.7-i2v",
Prompt: "continue the clip",
Image: "https://example.com/direct.png",
Images: []string{"https://example.com/images-first.png", "https://example.com/images-last.png"},
InputReference: "https://example.com/input-reference.png",
Metadata: map[string]interface{}{
"input": map[string]interface{}{
"media": []interface{}{
map[string]interface{}{
"type": "first_clip",
"url": "https://example.com/input.mp4",
},
},
},
},
}
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
require.NoError(t, err)
require.Equal(t, []AliVideoMedia{
{Type: "first_clip", URL: "https://example.com/input.mp4"},
}, aliReq.Input.Media)
require.Empty(t, aliReq.Input.ImgURL)
body, err := common.Marshal(aliReq)
require.NoError(t, err)
require.Contains(t, string(body), `"media"`)
require.NotContains(t, string(body), `"img_url"`)
}
func TestConvertToAliRequestWan27I2VRequiresMedia(t *testing.T) {
adaptor := &TaskAdaptor{}
req := relaycommon.TaskSubmitReq{
Model: "wan2.7-i2v",
Prompt: "animate without a frame",
}
_, err := adaptor.convertToAliRequest(testRelayInfo(), req)
require.Error(t, err)
require.True(t, strings.Contains(err.Error(), "requires image"))
}
func TestConvertToAliRequestWan25I2VKeepsLegacyImgURL(t *testing.T) {
adaptor := &TaskAdaptor{}
req := relaycommon.TaskSubmitReq{
Model: "wan2.5-i2v-preview",
Prompt: "animate the first frame",
Image: "https://example.com/first.png",
}
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
require.NoError(t, err)
require.Equal(t, "https://example.com/first.png", aliReq.Input.ImgURL)
require.Empty(t, aliReq.Input.Media)
body, err := common.Marshal(aliReq)
require.NoError(t, err)
require.Contains(t, string(body), `"img_url"`)
require.NotContains(t, string(body), `"media"`)
}
relay/channel/task/ali/constants.go
package ali package ali
var ModelList = []string{ var ModelList = []string{
"wan2.7-i2v", // 万相2.7图生视频（新input.media协议）
"wan2.7-t2v", // 万相2.7文生视频
"wan2.5-i2v-preview", // 万相2.5 preview（有声视频）推荐 "wan2.5-i2v-preview", // 万相2.5 preview（有声视频）推荐
"wan2.2-i2v-flash", // 万相2.2极速版（无声视频） "wan2.2-i2v-flash", // 万相2.2极速版（无声视频）
"wan2.2-i2v-plus", // 万相2.2专业版（无声视频） "wan2.2-i2v-plus", // 万相2.2专业版（无声视频）
......
relay/common/relay_utils.go
...@@ -139,6 +139,9 @@ func ValidateMultipartDirect(c *gin.Context, info *RelayInfo) *dto.TaskError { ...@@ -139,6 +139,9 @@ func ValidateMultipartDirect(c *gin.Context, info *RelayInfo) *dto.TaskError {
} }
if req.InputReference != "" { if req.InputReference != "" {
req.Images = []string{req.InputReference} req.Images = []string{req.InputReference}
} else if len(req.Images) == 0 && strings.TrimSpace(req.Image) != "" {
// 兼容单图上传
req.Images = []string{strings.TrimSpace(req.Image)}
} }
if strings.TrimSpace(req.Model) == "" { if strings.TrimSpace(req.Model) == "" {
......
relay/common/relay_utils_test.go 0 → 100644
package common
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/QuantumNous/new-api/constant"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
func TestValidateMultipartDirectNormalizesImageField(t *testing.T) {
gin.SetMode(gin.TestMode)
body := strings.NewReader(`{"model":"wan2.7-i2v","prompt":"animate","image":" https://example.com/first.png "}`)
request := httptest.NewRequest(http.MethodPost, "/v1/video/generations", body)
request.Header.Set("Content-Type", "application/json")
recorder := httptest.NewRecorder()
context, _ := gin.CreateTestContext(recorder)
context.Request = request
info := &RelayInfo{
TaskRelayInfo: &TaskRelayInfo{},
}
taskErr := ValidateMultipartDirect(context, info)
require.Nil(t, taskErr)
storedReq, err := GetTaskRequest(context)
require.NoError(t, err)
require.Equal(t, []string{"https://example.com/first.png"}, storedReq.Images)
require.Equal(t, constant.TaskActionGenerate, info.Action)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment