2026-06-02 12:38:01 +08:00
|
|
|
import {
|
|
|
|
|
analyzeProductImages,
|
|
|
|
|
buildProductSummary,
|
|
|
|
|
extractSellingPoints,
|
|
|
|
|
generateCreativeOptions,
|
|
|
|
|
generateStoryboard,
|
|
|
|
|
generateVideoPrompts,
|
|
|
|
|
checkCompliance,
|
|
|
|
|
type AdVideoUserConfig,
|
|
|
|
|
} from "../../api/adVideoPlanClient";
|
|
|
|
|
import { aiGenerationClient } from "../../api/aiGenerationClient";
|
|
|
|
|
import { waitForTask } from "../../api/taskSubscription";
|
|
|
|
|
import { resolveVideoRequestModel } from "../../utils/resolveVideoModel";
|
|
|
|
|
import type {
|
2026-06-03 12:16:33 +08:00
|
|
|
EcommerceVideoPlanProgress,
|
2026-06-02 12:38:01 +08:00
|
|
|
EcommerceVideoPlanResult,
|
|
|
|
|
EcommerceVideoSceneTask,
|
|
|
|
|
PlanStep,
|
|
|
|
|
} from "./ecommerceVideoTypes";
|
|
|
|
|
|
|
|
|
|
export interface PlanCallbacks {
|
|
|
|
|
onStepStart: (step: PlanStep) => void;
|
|
|
|
|
onStepDone: (step: PlanStep) => void;
|
2026-06-03 01:39:06 +08:00
|
|
|
onImagesUploaded?: (urls: string[]) => void;
|
2026-06-03 12:16:33 +08:00
|
|
|
onPartialProgress?: (progress: EcommerceVideoPlanProgress) => void;
|
2026-06-02 12:38:01 +08:00
|
|
|
signal?: AbortSignal;
|
2026-06-03 12:16:33 +08:00
|
|
|
/** Partial state from a previous run; steps with existing data are skipped. */
|
|
|
|
|
resumeFrom?: EcommerceVideoPlanProgress;
|
2026-06-02 12:38:01 +08:00
|
|
|
}
|
|
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
/**
|
|
|
|
|
* Run the full ad video planning pipeline.
|
|
|
|
|
* Supports resumption: if `resumeFrom` contains data for a step, that step is skipped.
|
|
|
|
|
* After each step, `onPartialProgress` fires so callers can persist intermediate state.
|
|
|
|
|
*/
|
2026-06-02 12:38:01 +08:00
|
|
|
export async function runVideoPlan(
|
|
|
|
|
imageDataUrls: string[],
|
|
|
|
|
manualText: string,
|
|
|
|
|
config: AdVideoUserConfig,
|
|
|
|
|
callbacks: PlanCallbacks,
|
|
|
|
|
): Promise<EcommerceVideoPlanResult> {
|
2026-06-03 12:16:33 +08:00
|
|
|
const { onStepStart, onStepDone, signal, resumeFrom = {} } = callbacks;
|
|
|
|
|
const progress: EcommerceVideoPlanProgress = { ...resumeFrom };
|
|
|
|
|
const emit = () => callbacks.onPartialProgress?.({ ...progress });
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: upload ──────────────────────────────────────
|
|
|
|
|
if (!progress.imageUrls?.length) {
|
|
|
|
|
onStepStart("upload");
|
|
|
|
|
const imageUrls: string[] = [];
|
|
|
|
|
const SUPPORTED_IMAGE_TYPES = new Set(["image/jpeg", "image/png", "image/webp", "image/gif"]);
|
|
|
|
|
for (const srcUrl of imageDataUrls) {
|
|
|
|
|
try {
|
|
|
|
|
const resp = await fetch(srcUrl);
|
|
|
|
|
const rawBlob = await resp.blob();
|
|
|
|
|
const mimeType = SUPPORTED_IMAGE_TYPES.has(rawBlob.type) ? rawBlob.type : "image/png";
|
|
|
|
|
const blob = rawBlob.type === mimeType ? rawBlob : new Blob([rawBlob], { type: mimeType });
|
|
|
|
|
const result = await aiGenerationClient.uploadAssetBinary(blob, { mimeType, scope: "ecommerce-product" });
|
|
|
|
|
imageUrls.push(result.url);
|
|
|
|
|
} catch {
|
|
|
|
|
// skip images that fail to upload
|
|
|
|
|
}
|
2026-06-02 16:16:09 +08:00
|
|
|
}
|
2026-06-03 12:16:33 +08:00
|
|
|
if (!imageUrls.length) throw new Error("图片上传失败,请检查图片格式或网络后重试");
|
|
|
|
|
progress.imageUrls = imageUrls;
|
|
|
|
|
onStepDone("upload");
|
|
|
|
|
callbacks.onImagesUploaded?.(imageUrls);
|
|
|
|
|
emit();
|
2026-06-02 12:38:01 +08:00
|
|
|
}
|
|
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: analyze ─────────────────────────────────────
|
|
|
|
|
if (progress.imageDescription === undefined) {
|
|
|
|
|
onStepStart("analyze");
|
|
|
|
|
progress.imageDescription = await analyzeProductImages(progress.imageUrls!, signal);
|
|
|
|
|
onStepDone("analyze");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: summary ─────────────────────────────────────
|
|
|
|
|
if (!progress.summary) {
|
|
|
|
|
onStepStart("summary");
|
|
|
|
|
progress.summary = await buildProductSummary(progress.imageDescription || "", manualText, signal);
|
|
|
|
|
onStepDone("summary");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: selling ─────────────────────────────────────
|
|
|
|
|
if (!progress.selling) {
|
|
|
|
|
onStepStart("selling");
|
|
|
|
|
progress.selling = await extractSellingPoints(progress.summary, signal);
|
|
|
|
|
onStepDone("selling");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: creative ────────────────────────────────────
|
|
|
|
|
if (!progress.creatives?.length) {
|
|
|
|
|
onStepStart("creative");
|
|
|
|
|
progress.creatives = await generateCreativeOptions(progress.selling, config, signal);
|
|
|
|
|
if (!progress.creatives.length) throw new Error("未能生成有效的广告创意");
|
|
|
|
|
onStepDone("creative");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: storyboard ──────────────────────────────────
|
|
|
|
|
if (!progress.storyboard) {
|
|
|
|
|
onStepStart("storyboard");
|
|
|
|
|
progress.storyboard = await generateStoryboard(progress.creatives[0], progress.summary, config, signal);
|
|
|
|
|
onStepDone("storyboard");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: prompts ─────────────────────────────────────
|
|
|
|
|
if (!progress.videoPrompts) {
|
|
|
|
|
onStepStart("prompts");
|
|
|
|
|
progress.videoPrompts = await generateVideoPrompts(progress.storyboard, progress.summary, signal);
|
|
|
|
|
onStepDone("prompts");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: compliance ──────────────────────────────────
|
|
|
|
|
if (!progress.compliance) {
|
|
|
|
|
onStepStart("compliance");
|
|
|
|
|
progress.compliance = await checkCompliance(progress.summary, progress.selling, progress.storyboard, signal);
|
|
|
|
|
onStepDone("compliance");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
return {
|
|
|
|
|
imageUrls: progress.imageUrls!,
|
|
|
|
|
imageDescription: progress.imageDescription,
|
|
|
|
|
summary: progress.summary!,
|
|
|
|
|
selling: progress.selling!,
|
|
|
|
|
creatives: progress.creatives!,
|
|
|
|
|
storyboard: progress.storyboard!,
|
|
|
|
|
videoPrompts: progress.videoPrompts!,
|
|
|
|
|
compliance: progress.compliance!,
|
|
|
|
|
};
|
2026-06-02 12:38:01 +08:00
|
|
|
}
|
|
|
|
|
|
2026-06-03 01:39:06 +08:00
|
|
|
export interface RenderSceneImageInput {
|
|
|
|
|
sceneId: number;
|
|
|
|
|
prompt: string;
|
|
|
|
|
aspectRatio: string;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface RenderImageCallbacks {
|
|
|
|
|
onSceneImageSubmitted: (sceneId: number, taskId: string) => void;
|
|
|
|
|
onSceneImageProgress: (sceneId: number, progress: number) => void;
|
|
|
|
|
onSceneImageCompleted: (sceneId: number, resultUrl: string) => void;
|
|
|
|
|
onSceneImageFailed: (sceneId: number, error: string) => void;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function renderSceneImage(
|
|
|
|
|
input: RenderSceneImageInput,
|
|
|
|
|
callbacks: RenderImageCallbacks,
|
|
|
|
|
abortRef: { current: boolean },
|
|
|
|
|
): Promise<void> {
|
|
|
|
|
const { taskId } = await aiGenerationClient.createImageTask({
|
|
|
|
|
model: "gpt-image-2",
|
|
|
|
|
prompt: input.prompt,
|
|
|
|
|
ratio: input.aspectRatio,
|
|
|
|
|
quality: "2K",
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
callbacks.onSceneImageSubmitted(input.sceneId, taskId);
|
|
|
|
|
|
|
|
|
|
const resultUrl = await waitForTask(taskId, {
|
|
|
|
|
abortRef,
|
|
|
|
|
onProgress: (e) => callbacks.onSceneImageProgress(input.sceneId, e.progress),
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (resultUrl) {
|
|
|
|
|
callbacks.onSceneImageCompleted(input.sceneId, resultUrl);
|
|
|
|
|
} else {
|
|
|
|
|
callbacks.onSceneImageFailed(input.sceneId, "图片生成未返回结果");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-02 12:38:01 +08:00
|
|
|
export interface RenderSceneInput {
|
|
|
|
|
sceneId: number;
|
|
|
|
|
prompt: string;
|
|
|
|
|
durationSeconds: number;
|
|
|
|
|
imageUrl: string;
|
|
|
|
|
aspectRatio: string;
|
|
|
|
|
resolution: string;
|
|
|
|
|
model?: string;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface RenderCallbacks {
|
|
|
|
|
onSceneSubmitted: (sceneId: number, taskId: string) => void;
|
|
|
|
|
onSceneProgress: (sceneId: number, progress: number) => void;
|
|
|
|
|
onSceneCompleted: (sceneId: number, resultUrl: string) => void;
|
|
|
|
|
onSceneFailed: (sceneId: number, error: string) => void;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function renderScene(
|
|
|
|
|
input: RenderSceneInput,
|
|
|
|
|
callbacks: RenderCallbacks,
|
|
|
|
|
abortRef: { current: boolean },
|
|
|
|
|
): Promise<void> {
|
|
|
|
|
const model = resolveVideoRequestModel({
|
|
|
|
|
model: input.model || "happyhorse-1.0",
|
|
|
|
|
referenceUrls: [input.imageUrl],
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
const { taskId } = await aiGenerationClient.createVideoTask({
|
|
|
|
|
model,
|
|
|
|
|
prompt: input.prompt,
|
|
|
|
|
ratio: input.aspectRatio,
|
|
|
|
|
duration: input.durationSeconds,
|
|
|
|
|
quality: input.resolution,
|
|
|
|
|
resolution: input.resolution,
|
2026-06-03 01:39:06 +08:00
|
|
|
frameMode: "start-end",
|
2026-06-02 12:38:01 +08:00
|
|
|
referenceUrls: [input.imageUrl],
|
|
|
|
|
hasReferenceVideo: false,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
callbacks.onSceneSubmitted(input.sceneId, taskId);
|
|
|
|
|
|
|
|
|
|
const resultUrl = await waitForTask(taskId, {
|
|
|
|
|
abortRef,
|
|
|
|
|
onProgress: (e) => callbacks.onSceneProgress(input.sceneId, e.progress),
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (resultUrl) {
|
|
|
|
|
callbacks.onSceneCompleted(input.sceneId, resultUrl);
|
|
|
|
|
} else {
|
|
|
|
|
callbacks.onSceneFailed(input.sceneId, "任务未返回结果");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function buildSceneTasks(
|
|
|
|
|
plan: EcommerceVideoPlanResult,
|
|
|
|
|
): EcommerceVideoSceneTask[] {
|
|
|
|
|
return plan.storyboard.scenes.map((scene) => {
|
2026-06-03 01:39:06 +08:00
|
|
|
const matchedPrompt = plan.videoPrompts.find((p) => p.scene_id === scene.scene_id);
|
2026-06-02 12:38:01 +08:00
|
|
|
return {
|
|
|
|
|
sceneId: scene.scene_id,
|
2026-06-03 01:39:06 +08:00
|
|
|
prompt: matchedPrompt?.positive_prompt || scene.visual_description,
|
2026-06-02 12:38:01 +08:00
|
|
|
durationSeconds: Number.parseInt(scene.duration, 10) || 5,
|
2026-06-03 01:39:06 +08:00
|
|
|
status: "idle" as const,
|
2026-06-02 12:38:01 +08:00
|
|
|
progress: 0,
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|