2026-06-02 12:38:01 +08:00
|
|
|
import {
|
|
|
|
|
analyzeProductImages,
|
|
|
|
|
buildProductSummary,
|
|
|
|
|
extractSellingPoints,
|
|
|
|
|
generateCreativeOptions,
|
|
|
|
|
generateStoryboard,
|
|
|
|
|
generateVideoPrompts,
|
|
|
|
|
checkCompliance,
|
|
|
|
|
type AdVideoUserConfig,
|
|
|
|
|
} from "../../api/adVideoPlanClient";
|
|
|
|
|
import { aiGenerationClient } from "../../api/aiGenerationClient";
|
|
|
|
|
import { waitForTask } from "../../api/taskSubscription";
|
|
|
|
|
import { resolveVideoRequestModel } from "../../utils/resolveVideoModel";
|
2026-06-03 20:19:07 +08:00
|
|
|
import { normalizeEcommerceImageMime } from "./ecommerceImageValidation";
|
2026-06-02 12:38:01 +08:00
|
|
|
import type {
|
2026-06-03 12:16:33 +08:00
|
|
|
EcommerceVideoPlanProgress,
|
2026-06-02 12:38:01 +08:00
|
|
|
EcommerceVideoPlanResult,
|
|
|
|
|
EcommerceVideoSceneTask,
|
|
|
|
|
PlanStep,
|
|
|
|
|
} from "./ecommerceVideoTypes";
|
|
|
|
|
|
|
|
|
|
export interface PlanCallbacks {
|
|
|
|
|
onStepStart: (step: PlanStep) => void;
|
|
|
|
|
onStepDone: (step: PlanStep) => void;
|
2026-06-03 01:39:06 +08:00
|
|
|
onImagesUploaded?: (urls: string[]) => void;
|
2026-06-03 20:19:07 +08:00
|
|
|
onUploadRejected?: (messages: string[]) => void;
|
2026-06-03 12:16:33 +08:00
|
|
|
onPartialProgress?: (progress: EcommerceVideoPlanProgress) => void;
|
2026-06-02 12:38:01 +08:00
|
|
|
signal?: AbortSignal;
|
2026-06-03 12:16:33 +08:00
|
|
|
/** Partial state from a previous run; steps with existing data are skipped. */
|
|
|
|
|
resumeFrom?: EcommerceVideoPlanProgress;
|
2026-06-02 12:38:01 +08:00
|
|
|
}
|
|
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
/**
|
|
|
|
|
* Run the full ad video planning pipeline.
|
|
|
|
|
* Supports resumption: if `resumeFrom` contains data for a step, that step is skipped.
|
|
|
|
|
* After each step, `onPartialProgress` fires so callers can persist intermediate state.
|
|
|
|
|
*/
|
2026-06-02 12:38:01 +08:00
|
|
|
export async function runVideoPlan(
|
|
|
|
|
imageDataUrls: string[],
|
|
|
|
|
manualText: string,
|
|
|
|
|
config: AdVideoUserConfig,
|
|
|
|
|
callbacks: PlanCallbacks,
|
|
|
|
|
): Promise<EcommerceVideoPlanResult> {
|
2026-06-03 12:16:33 +08:00
|
|
|
const { onStepStart, onStepDone, signal, resumeFrom = {} } = callbacks;
|
|
|
|
|
const progress: EcommerceVideoPlanProgress = { ...resumeFrom };
|
|
|
|
|
const emit = () => callbacks.onPartialProgress?.({ ...progress });
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: upload ──────────────────────────────────────
|
|
|
|
|
if (!progress.imageUrls?.length) {
|
|
|
|
|
onStepStart("upload");
|
|
|
|
|
const imageUrls: string[] = [];
|
2026-06-03 20:19:07 +08:00
|
|
|
const rejected: string[] = [];
|
2026-06-03 12:16:33 +08:00
|
|
|
for (const srcUrl of imageDataUrls) {
|
|
|
|
|
try {
|
|
|
|
|
const resp = await fetch(srcUrl);
|
|
|
|
|
const rawBlob = await resp.blob();
|
2026-06-03 20:19:07 +08:00
|
|
|
const mimeType = normalizeEcommerceImageMime(rawBlob.type);
|
2026-06-03 12:16:33 +08:00
|
|
|
const blob = rawBlob.type === mimeType ? rawBlob : new Blob([rawBlob], { type: mimeType });
|
2026-06-03 20:19:07 +08:00
|
|
|
const dataUrl = await new Promise<string>((resolve, reject) => {
|
|
|
|
|
const reader = new FileReader();
|
|
|
|
|
reader.onload = () => resolve(String(reader.result || ""));
|
|
|
|
|
reader.onerror = () => reject(reader.error || new Error("文件读取失败"));
|
|
|
|
|
reader.readAsDataURL(blob);
|
|
|
|
|
});
|
|
|
|
|
const result = await aiGenerationClient.uploadAsset({ dataUrl, mimeType, scope: "ecommerce-product" });
|
2026-06-03 12:16:33 +08:00
|
|
|
imageUrls.push(result.url);
|
2026-06-03 20:19:07 +08:00
|
|
|
} catch (err) {
|
|
|
|
|
rejected.push(err instanceof Error ? err.message : "图片上传失败");
|
2026-06-03 12:16:33 +08:00
|
|
|
}
|
2026-06-02 16:16:09 +08:00
|
|
|
}
|
2026-06-03 20:19:07 +08:00
|
|
|
if (rejected.length) {
|
|
|
|
|
progress.uploadWarnings = rejected;
|
|
|
|
|
callbacks.onUploadRejected?.(rejected);
|
|
|
|
|
}
|
2026-06-03 12:16:33 +08:00
|
|
|
if (!imageUrls.length) throw new Error("图片上传失败,请检查图片格式或网络后重试");
|
|
|
|
|
progress.imageUrls = imageUrls;
|
|
|
|
|
onStepDone("upload");
|
|
|
|
|
callbacks.onImagesUploaded?.(imageUrls);
|
|
|
|
|
emit();
|
2026-06-02 12:38:01 +08:00
|
|
|
}
|
|
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: analyze ─────────────────────────────────────
|
|
|
|
|
if (progress.imageDescription === undefined) {
|
|
|
|
|
onStepStart("analyze");
|
|
|
|
|
progress.imageDescription = await analyzeProductImages(progress.imageUrls!, signal);
|
|
|
|
|
onStepDone("analyze");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: summary ─────────────────────────────────────
|
|
|
|
|
if (!progress.summary) {
|
|
|
|
|
onStepStart("summary");
|
|
|
|
|
progress.summary = await buildProductSummary(progress.imageDescription || "", manualText, signal);
|
|
|
|
|
onStepDone("summary");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: selling ─────────────────────────────────────
|
|
|
|
|
if (!progress.selling) {
|
|
|
|
|
onStepStart("selling");
|
|
|
|
|
progress.selling = await extractSellingPoints(progress.summary, signal);
|
|
|
|
|
onStepDone("selling");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: creative ────────────────────────────────────
|
|
|
|
|
if (!progress.creatives?.length) {
|
|
|
|
|
onStepStart("creative");
|
|
|
|
|
progress.creatives = await generateCreativeOptions(progress.selling, config, signal);
|
|
|
|
|
if (!progress.creatives.length) throw new Error("未能生成有效的广告创意");
|
|
|
|
|
onStepDone("creative");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: storyboard ──────────────────────────────────
|
|
|
|
|
if (!progress.storyboard) {
|
|
|
|
|
onStepStart("storyboard");
|
|
|
|
|
progress.storyboard = await generateStoryboard(progress.creatives[0], progress.summary, config, signal);
|
|
|
|
|
onStepDone("storyboard");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: prompts ─────────────────────────────────────
|
|
|
|
|
if (!progress.videoPrompts) {
|
|
|
|
|
onStepStart("prompts");
|
|
|
|
|
progress.videoPrompts = await generateVideoPrompts(progress.storyboard, progress.summary, signal);
|
|
|
|
|
onStepDone("prompts");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
// ── Step: compliance ──────────────────────────────────
|
|
|
|
|
if (!progress.compliance) {
|
|
|
|
|
onStepStart("compliance");
|
|
|
|
|
progress.compliance = await checkCompliance(progress.summary, progress.selling, progress.storyboard, signal);
|
|
|
|
|
onStepDone("compliance");
|
|
|
|
|
emit();
|
|
|
|
|
}
|
2026-06-02 12:38:01 +08:00
|
|
|
|
2026-06-03 12:16:33 +08:00
|
|
|
return {
|
|
|
|
|
imageUrls: progress.imageUrls!,
|
|
|
|
|
imageDescription: progress.imageDescription,
|
|
|
|
|
summary: progress.summary!,
|
|
|
|
|
selling: progress.selling!,
|
|
|
|
|
creatives: progress.creatives!,
|
|
|
|
|
storyboard: progress.storyboard!,
|
|
|
|
|
videoPrompts: progress.videoPrompts!,
|
|
|
|
|
compliance: progress.compliance!,
|
|
|
|
|
};
|
2026-06-02 12:38:01 +08:00
|
|
|
}
|
|
|
|
|
|
2026-06-03 01:39:06 +08:00
|
|
|
export interface RenderSceneImageInput {
|
|
|
|
|
sceneId: number;
|
|
|
|
|
prompt: string;
|
|
|
|
|
aspectRatio: string;
|
2026-06-03 23:52:25 +08:00
|
|
|
productImageUrls: string[];
|
2026-06-03 01:39:06 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface RenderImageCallbacks {
|
|
|
|
|
onSceneImageSubmitted: (sceneId: number, taskId: string) => void;
|
|
|
|
|
onSceneImageProgress: (sceneId: number, progress: number) => void;
|
|
|
|
|
onSceneImageCompleted: (sceneId: number, resultUrl: string) => void;
|
|
|
|
|
onSceneImageFailed: (sceneId: number, error: string) => void;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function renderSceneImage(
|
|
|
|
|
input: RenderSceneImageInput,
|
|
|
|
|
callbacks: RenderImageCallbacks,
|
|
|
|
|
abortRef: { current: boolean },
|
|
|
|
|
): Promise<void> {
|
|
|
|
|
const { taskId } = await aiGenerationClient.createImageTask({
|
|
|
|
|
model: "gpt-image-2",
|
|
|
|
|
prompt: input.prompt,
|
|
|
|
|
ratio: input.aspectRatio,
|
|
|
|
|
quality: "2K",
|
2026-06-03 23:52:25 +08:00
|
|
|
referenceUrls: input.productImageUrls,
|
2026-06-03 01:39:06 +08:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
callbacks.onSceneImageSubmitted(input.sceneId, taskId);
|
|
|
|
|
|
|
|
|
|
const resultUrl = await waitForTask(taskId, {
|
|
|
|
|
abortRef,
|
|
|
|
|
onProgress: (e) => callbacks.onSceneImageProgress(input.sceneId, e.progress),
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (resultUrl) {
|
|
|
|
|
callbacks.onSceneImageCompleted(input.sceneId, resultUrl);
|
|
|
|
|
} else {
|
|
|
|
|
callbacks.onSceneImageFailed(input.sceneId, "图片生成未返回结果");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-02 12:38:01 +08:00
|
|
|
export interface RenderSceneInput {
|
|
|
|
|
sceneId: number;
|
|
|
|
|
prompt: string;
|
|
|
|
|
durationSeconds: number;
|
|
|
|
|
imageUrl: string;
|
2026-06-03 23:52:25 +08:00
|
|
|
productImageUrls: string[];
|
2026-06-02 12:38:01 +08:00
|
|
|
aspectRatio: string;
|
|
|
|
|
resolution: string;
|
|
|
|
|
model?: string;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface RenderCallbacks {
|
|
|
|
|
onSceneSubmitted: (sceneId: number, taskId: string) => void;
|
|
|
|
|
onSceneProgress: (sceneId: number, progress: number) => void;
|
|
|
|
|
onSceneCompleted: (sceneId: number, resultUrl: string) => void;
|
|
|
|
|
onSceneFailed: (sceneId: number, error: string) => void;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function renderScene(
|
|
|
|
|
input: RenderSceneInput,
|
|
|
|
|
callbacks: RenderCallbacks,
|
|
|
|
|
abortRef: { current: boolean },
|
|
|
|
|
): Promise<void> {
|
2026-06-03 23:52:25 +08:00
|
|
|
const allReferenceUrls = [...input.productImageUrls, input.imageUrl];
|
2026-06-02 12:38:01 +08:00
|
|
|
const model = resolveVideoRequestModel({
|
|
|
|
|
model: input.model || "happyhorse-1.0",
|
2026-06-03 23:52:25 +08:00
|
|
|
referenceUrls: allReferenceUrls,
|
2026-06-02 12:38:01 +08:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
const { taskId } = await aiGenerationClient.createVideoTask({
|
|
|
|
|
model,
|
|
|
|
|
prompt: input.prompt,
|
|
|
|
|
ratio: input.aspectRatio,
|
|
|
|
|
duration: input.durationSeconds,
|
|
|
|
|
quality: input.resolution,
|
|
|
|
|
resolution: input.resolution,
|
2026-06-03 01:39:06 +08:00
|
|
|
frameMode: "start-end",
|
2026-06-03 23:52:25 +08:00
|
|
|
referenceUrls: allReferenceUrls,
|
2026-06-02 12:38:01 +08:00
|
|
|
hasReferenceVideo: false,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
callbacks.onSceneSubmitted(input.sceneId, taskId);
|
|
|
|
|
|
|
|
|
|
const resultUrl = await waitForTask(taskId, {
|
|
|
|
|
abortRef,
|
|
|
|
|
onProgress: (e) => callbacks.onSceneProgress(input.sceneId, e.progress),
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (resultUrl) {
|
|
|
|
|
callbacks.onSceneCompleted(input.sceneId, resultUrl);
|
|
|
|
|
} else {
|
|
|
|
|
callbacks.onSceneFailed(input.sceneId, "任务未返回结果");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function buildSceneTasks(
|
|
|
|
|
plan: EcommerceVideoPlanResult,
|
|
|
|
|
): EcommerceVideoSceneTask[] {
|
|
|
|
|
return plan.storyboard.scenes.map((scene) => {
|
2026-06-03 01:39:06 +08:00
|
|
|
const matchedPrompt = plan.videoPrompts.find((p) => p.scene_id === scene.scene_id);
|
2026-06-02 12:38:01 +08:00
|
|
|
return {
|
|
|
|
|
sceneId: scene.scene_id,
|
2026-06-03 01:39:06 +08:00
|
|
|
prompt: matchedPrompt?.positive_prompt || scene.visual_description,
|
2026-06-02 12:38:01 +08:00
|
|
|
durationSeconds: Number.parseInt(scene.duration, 10) || 5,
|
2026-06-03 01:39:06 +08:00
|
|
|
status: "idle" as const,
|
2026-06-02 12:38:01 +08:00
|
|
|
progress: 0,
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|