Commit 3b465226 by Calcium-Ion Committed by GitHub

feat(ali): support wan3.0 all-in-one video models

* feat(ali): support wan3.0 all-in-one video models

Extend the Alibaba task plugin for wan3.0-video / wan3.0-video-prime
using DashScope media, resolution/ratio, duration rules, and billing ratios.

* fix(ali): make wan3.0 smart duration billable and alias-safe

Review fixes on top of #7240 (qiuliw):

- duration -1 never reached the plugin: the host rejects negative
  canonical duration/seconds facts before any hook runs, on every
  entry point. Decoders now normalize -1 into an auto_duration marker;
  convert emits -1 upstream and bills 30s up front; non-wan3.0 models
  reject the marker instead of silently defaulting to 5s.
- extractUsageOnComplete read output.duration/output.resolution, but
  wan3.0 reports usage.output_video_duration and numeric usage.SR, so
  smart-duration and resolution settlement never reconciled. Read the
  usage block first, keep the legacy output fields as fallback.
- convert keyed default resolution on the client model name, so a
  channel-mapped alias fell to 720P while the direct request got 1080P.
  Every model-shaped decision in convert now uses ctx.upstreamModel.
- Unknown wan3.0 size values were silently coerced to 1080P; reject them.
- Image-only openai_responses input was accepted for every model,
  regressing t2v models into pre-consume then upstream rejection.
  Restore the guard and allow image-only for i2v and wan3.0 only.
- Native passthrough dropped wan3.0 parameters (ratio, audio) unless
  media was present; forward them via metadata for wan3.0.
- Hoist the triplicated size-to-resolution map; add wan3.0 contract tests.

---------

Co-authored-by: qiuliw <a1807191473@qgmail.com>
parent 0c76e4da
package plugins_test
import (
"testing"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/pkg/jsplugin"
builtinplugins "github.com/QuantumNous/new-api/plugins"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAlibabaWan3(t *testing.T) {
source, err := builtinplugins.Source("alibaba")
require.NoError(t, err)
registry := jsplugin.NewRegistry()
plugin, err := registry.RegisterFactory(source, jsplugin.Options{Key: "alibaba"})
require.NoError(t, err)
roundTrip := func(t *testing.T, value any) map[string]any {
encoded, marshalErr := common.Marshal(value)
require.NoError(t, marshalErr)
var decoded map[string]any
require.NoError(t, common.Unmarshal(encoded, &decoded))
return decoded
}
submitCtx := func(model, upstream string, body map[string]any) map[string]any {
return map[string]any{"model": model, "upstreamModel": upstream, "baseUrl": "https://dashscope.aliyuncs.com", "apiKey": "k", "requestBody": body}
}
usageCtx := func(purpose string, body map[string]any) map[string]any {
return map[string]any{"model": "wan3.0-video", "upstreamModel": "wan3.0-video", "usagePurpose": purpose, "requestBody": body}
}
decodeResponses := func(model string, body map[string]any) (map[string]any, error) {
value, callErr := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_responses", "decodeRequest"}, map[string]any{"model": model, "body": map[string]any{"kind": "json", "value": body}, "stream": false})
if callErr != nil {
return nil, callErr
}
return roundTrip(t, value), nil
}
t.Run("duration -1 becomes the auto_duration marker so the host accepts the body", func(t *testing.T) {
resolved, callErr := decodeResponses("wan3.0-video", map[string]any{"model": "wan3.0-video", "input": "a cat", "duration": -1})
require.NoError(t, callErr)
requestBody := resolved["requestBody"].(map[string]any)
assert.Equal(t, true, requestBody["auto_duration"])
assert.NotContains(t, requestBody, "duration")
value, callErr := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_video", "decodeRequest"}, map[string]any{"model": "wan3.0-video", "body": map[string]any{"kind": "json", "value": map[string]any{"model": "wan3.0-video", "prompt": "a cat", "seconds": -1}}})
require.NoError(t, callErr)
requestBody = roundTrip(t, value)["requestBody"].(map[string]any)
assert.Equal(t, true, requestBody["auto_duration"])
assert.NotContains(t, requestBody, "seconds")
assert.NotContains(t, requestBody, "duration")
value, callErr = plugin.Engine.CallPath(t.Context(), "native", []string{"createVideoTask"}, map[string]any{"body": map[string]any{"kind": "json", "value": map[string]any{
"model": "wan3.0-video", "input": map[string]any{"prompt": "a cat"}, "parameters": map[string]any{"duration": -1, "ratio": "16:9"},
}}})
require.NoError(t, callErr)
requestBody = roundTrip(t, value)["requestBody"].(map[string]any)
assert.Equal(t, true, requestBody["auto_duration"])
assert.NotContains(t, requestBody, "duration")
parameters := requestBody["metadata"].(map[string]any)["parameters"].(map[string]any)
assert.Equal(t, "16:9", parameters["ratio"])
assert.NotContains(t, parameters, "duration")
})
t.Run("auto_duration submits -1 upstream and bills 30 seconds up front", func(t *testing.T) {
body := map[string]any{"model": "wan3.0-video", "prompt": "a cat", "auto_duration": true}
value, callErr := plugin.Engine.Call(t.Context(), "buildSubmitRequest", submitCtx("wan3.0-video", "wan3.0-video", body))
require.NoError(t, callErr)
parameters := roundTrip(t, value)["body"].(map[string]any)["parameters"].(map[string]any)
assert.Equal(t, float64(-1), parameters["duration"])
value, callErr = plugin.Engine.Call(t.Context(), "extractUsage", usageCtx("facts", body))
require.NoError(t, callErr)
assert.Equal(t, map[string]any{"seconds": float64(30), "resolution": "1080P"}, roundTrip(t, value))
value, callErr = plugin.Engine.Call(t.Context(), "extractUsage", usageCtx("billing_ratios", body))
require.NoError(t, callErr)
assert.Equal(t, float64(30), roundTrip(t, value)["seconds"])
_, callErr = plugin.Engine.Call(t.Context(), "buildSubmitRequest", submitCtx("wan2.7-t2v", "wan2.7-t2v", map[string]any{"model": "wan2.7-t2v", "prompt": "a cat", "auto_duration": true}))
require.ErrorContains(t, callErr, "only supported by wan3.0")
})
t.Run("channel-mapped alias resolves defaults from the upstream model", func(t *testing.T) {
direct, callErr := plugin.Engine.Call(t.Context(), "buildSubmitRequest", submitCtx("wan3.0-video", "wan3.0-video", map[string]any{"model": "wan3.0-video", "prompt": "a cat"}))
require.NoError(t, callErr)
alias, callErr := plugin.Engine.Call(t.Context(), "buildSubmitRequest", submitCtx("my-wan3", "wan3.0-video", map[string]any{"model": "my-wan3", "prompt": "a cat"}))
require.NoError(t, callErr)
assert.Equal(t, roundTrip(t, direct)["body"], roundTrip(t, alias)["body"])
assert.Equal(t, "1080P", roundTrip(t, alias)["body"].(map[string]any)["parameters"].(map[string]any)["resolution"])
})
t.Run("size maps to a resolution tier and unknown sizes are rejected", func(t *testing.T) {
value, callErr := plugin.Engine.Call(t.Context(), "buildSubmitRequest", submitCtx("wan3.0-video", "wan3.0-video", map[string]any{"model": "wan3.0-video", "prompt": "a cat", "size": "1280*720"}))
require.NoError(t, callErr)
parameters := roundTrip(t, value)["body"].(map[string]any)["parameters"].(map[string]any)
assert.Equal(t, "720P", parameters["resolution"])
assert.NotContains(t, parameters, "size")
assert.Equal(t, "adaptive", parameters["ratio"])
_, callErr = plugin.Engine.Call(t.Context(), "buildSubmitRequest", submitCtx("wan3.0-video", "wan3.0-video", map[string]any{"model": "wan3.0-video", "prompt": "a cat", "size": "1000*1000"}))
require.ErrorContains(t, callErr, "invalid size")
_, callErr = plugin.Engine.Call(t.Context(), "buildSubmitRequest", submitCtx("wan3.0-video", "wan3.0-video", map[string]any{"model": "wan3.0-video", "prompt": "a cat", "duration": 31}))
require.ErrorContains(t, callErr, "between 2 and 30")
})
t.Run("image-only input stays rejected for t2v models and accepted for wan3.0", func(t *testing.T) {
imageOnly := []any{map[string]any{"type": "input_image", "image_url": "https://cdn.example/first.png"}}
_, callErr := decodeResponses("wan2.7-t2v", map[string]any{"model": "wan2.7-t2v", "input": imageOnly})
require.ErrorContains(t, callErr, "input is required")
resolved, callErr := decodeResponses("wan3.0-video", map[string]any{"model": "wan3.0-video", "input": imageOnly})
require.NoError(t, callErr)
assert.Equal(t, "image_to_video", resolved["action"])
value, callErr := plugin.Engine.Call(t.Context(), "buildSubmitRequest", submitCtx("wan3.0-video", "wan3.0-video", map[string]any{"model": "wan3.0-video", "prompt": "", "images": []any{"https://cdn.example/first.png"}}))
require.NoError(t, callErr)
input := roundTrip(t, value)["body"].(map[string]any)["input"].(map[string]any)
assert.Equal(t, []any{map[string]any{"type": "first_frame", "url": "https://cdn.example/first.png"}}, input["media"])
assert.NotContains(t, input, "img_url")
})
t.Run("completion facts read the wan3.0 usage block", func(t *testing.T) {
value, callErr := plugin.Engine.Call(t.Context(), "extractUsageOnComplete", map[string]any{}, map[string]any{}, map[string]any{
"output": map[string]any{"task_status": "SUCCEEDED", "video_url": "https://upstream.example/v.mp4"},
"usage": map[string]any{"video_count": 1, "duration": 7.5, "output_video_duration": 7.5, "SR": 720, "ratio": "16:9"},
})
require.NoError(t, callErr)
assert.Equal(t, map[string]any{"seconds": 7.5, "resolution": "720P"}, roundTrip(t, value))
value, callErr = plugin.Engine.Call(t.Context(), "extractUsageOnComplete", map[string]any{}, map[string]any{}, map[string]any{
"output": map[string]any{"task_status": "SUCCEEDED", "duration": 5, "resolution": "1080p"},
})
require.NoError(t, callErr)
assert.Equal(t, map[string]any{"seconds": float64(5), "resolution": "1080P"}, roundTrip(t, value))
})
}
......@@ -7,10 +7,12 @@ export const meta = {
en: "Alibaba Cloud Bailian Wanxiang video generation (text-to-video and image-to-video)",
zh: "阿里云百炼万相视频生成(文生视频、图生视频)",
},
version: "1.0.1",
version: "1.1.0",
author: { name: "QuantumNous" },
channelTypes: [17],
models: [
"wan3.0-video",
"wan3.0-video-prime",
"wan2.7-i2v",
"wan2.7-t2v",
"wan2.5-t2v-preview",
......@@ -59,6 +61,41 @@ function secondImage(req) {
return "";
}
const SIZE_TO_RESOLUTION = {
"832*480": "480P",
"480*832": "480P",
"624*624": "480P",
"1280*720": "720P",
"720*1280": "720P",
"960*960": "720P",
"1088*832": "720P",
"832*1088": "720P",
"1920*1080": "1080P",
"1080*1920": "1080P",
"1440*1440": "1080P",
"1632*1248": "1080P",
"1248*1632": "1080P",
};
// The host rejects negative canonical duration/seconds facts before any hook
// runs, so wan3.0's "-1 = smart duration" sentinel travels as a boolean marker.
function normalizeAutoDuration(req) {
let auto = req.auto_duration === true;
for (const key of ["duration", "seconds"]) {
if (Number(req[key]) === -1) {
auto = true;
delete req[key];
}
}
const parameters = req.metadata && req.metadata.parameters;
if (parameters && Number(parameters.duration) === -1) {
auto = true;
delete parameters.duration;
}
if (auto) req.auto_duration = true;
return req;
}
function normalizeResolution(value) {
let resolution = String(value || "").toUpperCase();
if (!resolution.endsWith("P")) resolution += "P";
......@@ -74,18 +111,26 @@ function convert(ctx) {
const parameters = { prompt_extend: true, duration: 5 };
if (req.size) {
if (String(req.model).includes("t2v") && !String(req.size).includes("*")) throw new Error("invalid size: " + req.size + ", example: 1920*1080");
if (String(upstreamModel).includes("t2v") && !String(req.size).includes("*")) throw new Error("invalid size: " + req.size + ", example: 1920*1080");
if (String(req.size).includes("*")) parameters.size = req.size;
else parameters.resolution = normalizeResolution(req.size);
} else if (String(req.model).includes("t2v")) {
parameters.size = String(req.model).startsWith("wan2.5") || String(req.model).startsWith("wan2.2") ? "1920*1080" : "1280*720";
} else if (String(req.model).startsWith("wan2.6") || String(req.model).startsWith("wan2.5") || String(req.model).startsWith("wan2.2-i2v-plus")) {
} else if (String(upstreamModel).includes("t2v")) {
parameters.size = String(upstreamModel).startsWith("wan2.5") || String(upstreamModel).startsWith("wan2.2") ? "1920*1080" : "1280*720";
} else if (
String(upstreamModel).startsWith("wan2.6") ||
String(upstreamModel).startsWith("wan2.5") ||
String(upstreamModel).startsWith("wan2.2-i2v-plus") ||
String(upstreamModel).startsWith("wan3.0")
) {
parameters.resolution = "1080P";
} else {
parameters.resolution = "720P";
}
if (Number(req.duration) > 0) parameters.duration = Number(req.duration);
if (req.auto_duration === true) {
if (!String(upstreamModel).startsWith("wan3.0")) throw new Error("duration -1 (smart duration) is only supported by wan3.0 models");
parameters.duration = -1;
} else if (Number(req.duration) > 0) parameters.duration = Number(req.duration);
else if (req.seconds) {
const seconds = Number(req.seconds);
if (!Number.isInteger(seconds)) throw new Error("convert seconds to int failed");
......@@ -99,16 +144,32 @@ function convert(ctx) {
if (model !== upstreamModel) throw new Error("can't change model with metadata");
const body = { model: model, input: input, parameters: parameters };
if (String(model).startsWith("wan2.7-i2v")) {
if (String(model).startsWith("wan2.7-i2v") || String(model).startsWith("wan3.0-video")) {
if (!Array.isArray(input.media) || input.media.length === 0) {
input.media = [];
const first = trimmed(input.first_frame_url) || trimmed(input.img_url) || firstImage(req);
const last = trimmed(input.last_frame_url) || secondImage(req);
if (first) input.media.push({ type: "first_frame", url: first });
if (last) input.media.push({ type: "last_frame", url: last });
if (trimmed(input.audio_url)) input.media.push({ type: "driving_audio", url: input.audio_url });
if (String(model).startsWith("wan2.7-i2v") && trimmed(input.audio_url)) input.media.push({ type: "driving_audio", url: input.audio_url });
}
if (String(model).startsWith("wan2.7-i2v") && input.media.length === 0)
throw new Error("wan2.7-i2v requires image, images, input_reference, or input.media");
if (String(model).startsWith("wan3.0-video")) {
if (input.media.length === 0) delete input.media;
if (!trimmed(input.prompt) && !(Array.isArray(input.media) && input.media.length)) throw new Error("wan3.0-video requires prompt or input.media");
if (parameters.size) {
const mapped = SIZE_TO_RESOLUTION[parameters.size];
if (!mapped) throw new Error("invalid size: " + parameters.size + ", wan3.0 accepts resolution 480P, 720P, or 1080P");
parameters.resolution = mapped;
delete parameters.size;
}
if (!parameters.resolution) parameters.resolution = "1080P";
if (!parameters.ratio) parameters.ratio = "adaptive";
const duration = Number(parameters.duration);
if (duration !== -1 && (!Number.isInteger(duration) || duration < 2 || duration > 30))
throw new Error("wan3.0 duration must be -1 or an integer between 2 and 30");
}
if (input.media.length === 0) throw new Error("wan2.7-i2v requires image, images, input_reference, or input.media");
delete input.img_url;
delete input.first_frame_url;
delete input.last_frame_url;
......@@ -117,29 +178,15 @@ function convert(ctx) {
if (!parameters.prompt_extend) delete parameters.prompt_extend;
if (!parameters.watermark) delete parameters.watermark;
if (!parameters.seed) delete parameters.seed;
for (const key of ["resolution", "size"]) if (!parameters[key]) delete parameters[key];
for (const key of ["resolution", "size", "ratio"]) if (!parameters[key]) delete parameters[key];
return body;
}
function resolutionRatio(body) {
let resolution = body.parameters.size
? {
"832*480": "480P",
"480*832": "480P",
"624*624": "480P",
"1280*720": "720P",
"720*1280": "720P",
"960*960": "720P",
"1088*832": "720P",
"832*1088": "720P",
"1920*1080": "1080P",
"1080*1920": "1080P",
"1440*1440": "1080P",
"1632*1248": "1080P",
"1248*1632": "1080P",
}[body.parameters.size]
: normalizeResolution(body.parameters.resolution);
let resolution = body.parameters.size ? SIZE_TO_RESOLUTION[body.parameters.size] : normalizeResolution(body.parameters.resolution);
const ratios = {
"wan3.0-video": { "480P": 1, "720P": 2, "1080P": 4 },
"wan3.0-video-prime": { "480P": 1, "720P": 2, "1080P": 4 },
"wan2.6-i2v": { "720P": 1, "1080P": 1 / 0.6 },
"wan2.5-t2v-preview": { "480P": 1, "720P": 2, "1080P": 1 / 0.3 },
"wan2.2-t2v-plus": { "480P": 1, "1080P": 5 },
......@@ -205,7 +252,7 @@ export function buildSubmitRequest(ctx) {
method: "POST",
headers: { Authorization: "Bearer " + ctx.apiKey, "Content-Type": "application/json", "X-DashScope-Async": "enable" },
body: body,
action: firstImage(ctx.requestBody) ? "image_to_video" : "text_to_video",
action: firstImage(ctx.requestBody) || (body.input && Array.isArray(body.input.media) && body.input.media.length) ? "image_to_video" : "text_to_video",
};
}
......@@ -218,39 +265,26 @@ export function parseSubmitResponse(ctx, resp) {
export function extractUsage(ctx) {
const body = convert(ctx);
const duration = Number(body.parameters.duration);
const seconds = Math.min(duration === -1 ? 30 : duration, 3600);
if (ctx.usagePurpose === "billing_ratios") {
const ratios = { seconds: Math.min(Number(body.parameters.duration), 3600) };
const ratios = { seconds: seconds };
const resolution = resolutionRatio(body);
if (resolution && resolution.value !== undefined) ratios[resolution.key] = resolution.value;
return ratios;
}
let resolution = body.parameters.size
? {
"832*480": "480P",
"480*832": "480P",
"624*624": "480P",
"1280*720": "720P",
"720*1280": "720P",
"960*960": "720P",
"1088*832": "720P",
"832*1088": "720P",
"1920*1080": "1080P",
"1080*1920": "1080P",
"1440*1440": "1080P",
"1632*1248": "1080P",
"1248*1632": "1080P",
}[body.parameters.size]
: normalizeResolution(body.parameters.resolution);
if (!["480P", "720P", "1080P"].includes(resolution)) resolution = "720P";
return { seconds: Math.min(Number(body.parameters.duration), 3600), resolution: resolution };
let resolution = body.parameters.size ? SIZE_TO_RESOLUTION[body.parameters.size] : normalizeResolution(body.parameters.resolution);
if (!["480P", "720P", "1080P"].includes(resolution)) resolution = String(body.model).startsWith("wan3.0") ? "1080P" : "720P";
return { seconds: seconds, resolution: resolution };
}
export function extractUsageOnComplete(task, taskResult, body) {
const output = (body && body.output) || {};
const usage = (body && body.usage) || {};
const facts = {};
const seconds = Number(output.duration || output.duration_seconds || 0);
const seconds = Number(usage.output_video_duration || usage.duration || output.duration || output.duration_seconds || 0);
if (Number.isFinite(seconds) && seconds > 0) facts.seconds = Math.min(seconds, 3600);
const resolution = normalizeResolution(output.resolution || "");
const resolution = normalizeResolution(usage.SR || output.resolution || "");
if (["480P", "720P", "1080P"].includes(resolution)) facts.resolution = resolution;
return facts;
}
......@@ -297,17 +331,23 @@ export const native = {
const req = ctx.body.value,
input = req.input || {},
parameters = req.parameters || {};
const requestBody = {
model: req.model,
prompt: input.prompt || "",
image: input.img_url,
duration: parameters.duration,
size: parameters.size || parameters.resolution,
};
const hasMedia = Array.isArray(input.media) && input.media.length > 0;
if (hasMedia || String(req.model).startsWith("wan3.0")) {
requestBody.metadata = { input: hasMedia ? { media: input.media } : {}, parameters: Object.assign({}, parameters) };
}
normalizeAutoDuration(requestBody);
return {
kind: "submit",
model: req.model,
action: input.img_url ? "image_to_video" : "text_to_video",
requestBody: {
model: req.model,
prompt: input.prompt || "",
image: input.img_url,
duration: parameters.duration,
size: parameters.size || parameters.resolution,
},
action: input.img_url || hasMedia ? "image_to_video" : "text_to_video",
requestBody: requestBody,
};
},
taskCreated: function (ctx, task) {
......@@ -347,8 +387,12 @@ export const protocols = {
if (Object.prototype.hasOwnProperty.call(req, key)) requestBody[key] = req[key];
}
if (Object.prototype.hasOwnProperty.call(req, "metadata")) requestBody.metadata = req.metadata;
if (!prompt && (!model.includes("i2v") || !firstImage(requestBody))) throw new Error("input is required");
return { kind: "submit", model: model, action: firstImage(requestBody) ? "image_to_video" : "text_to_video", requestBody: requestBody };
normalizeAutoDuration(requestBody);
const hasMetaMedia = req.metadata && req.metadata.input && Array.isArray(req.metadata.input.media) && req.metadata.input.media.length > 0;
const upstream = String(ctx.upstreamModel || model);
const acceptsImageOnly = upstream.includes("i2v") || upstream.startsWith("wan3.0");
if (!prompt && !hasMetaMedia && !(acceptsImageOnly && firstImage(requestBody))) throw new Error("input is required");
return { kind: "submit", model: model, action: firstImage(requestBody) || hasMetaMedia ? "image_to_video" : "text_to_video", requestBody: requestBody };
},
renderEvents: function (ctx, task, previousState) {
const status = String(task.status || "UNKNOWN").toUpperCase();
......@@ -422,11 +466,12 @@ export const protocols = {
else if (req.duration !== undefined) req.seconds = Number(req.duration);
if (req.duration !== undefined) req.duration = Number(req.duration);
} else throw new Error("JSON or multipart body required");
const requestBody = normalizeAutoDuration(Object.assign({}, req, { model: ctx.model }));
return {
kind: "submit",
model: ctx.model,
action: firstImage(req) ? "image_to_video" : "text_to_video",
requestBody: Object.assign({}, req, { model: ctx.model }),
action: firstImage(requestBody) ? "image_to_video" : "text_to_video",
requestBody: requestBody,
};
},
render: function (ctx, task) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment