538 lines
17 KiB
TypeScript
538 lines
17 KiB
TypeScript
import {
|
||
analyzeProductImages,
|
||
buildProductSummary,
|
||
extractSellingPoints,
|
||
generateCreativeOptions,
|
||
generateStoryboard,
|
||
generateVideoPrompts,
|
||
checkCompliance,
|
||
type AdVideoUserConfig,
|
||
} from "../../api/adVideoPlanClient";
|
||
import { aiGenerationClient } from "../../api/aiGenerationClient";
|
||
import { serverRequest } from "../../api/serverConnection";
|
||
import { waitForTask } from "../../api/taskSubscription";
|
||
import { ENTERPRISE_DEFAULT_VIDEO_MODEL } from "../../utils/enterpriseVideoPolicy";
|
||
import { resolveVideoRequestModel } from "../../utils/resolveVideoModel";
|
||
import { ecommerceOssScopes } from "./ecommerceGenerationPersistence";
|
||
import { normalizeEcommerceImageMime } from "./ecommerceImageValidation";
|
||
import type {
|
||
EcommerceVideoPlanProgress,
|
||
EcommerceVideoPlanResult,
|
||
EcommerceVideoSceneTask,
|
||
PlanStep,
|
||
} from "./ecommerceVideoTypes";
|
||
|
||
type UploadAssetByUrl = typeof aiGenerationClient.uploadAssetByUrl;
|
||
|
||
interface DurableMediaUrl {
|
||
url: string | null;
|
||
originalUrl?: string | null;
|
||
ossKey?: string | null;
|
||
}
|
||
|
||
const TEMP_MEDIA_HOST_RE = /^file\d*\.aitohumanize\.com$/i;
|
||
const OSS_MEDIA_HOST_RE = /\.oss-[^.]+\.aliyuncs\.com$/i;
|
||
|
||
function isTemporaryProviderUrl(url: string): boolean {
|
||
try {
|
||
return TEMP_MEDIA_HOST_RE.test(new URL(url).hostname);
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function isDurableOssUrl(url: string): boolean {
|
||
try {
|
||
const parsed = new URL(url);
|
||
return parsed.protocol === "https:" && OSS_MEDIA_HOST_RE.test(parsed.hostname);
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function getMediaExtension(url: string, mimeType: string): string {
|
||
const normalizedMime = mimeType.split(";")[0]?.trim().toLowerCase();
|
||
if (normalizedMime === "image/jpeg") return "jpg";
|
||
if (normalizedMime === "image/png") return "png";
|
||
if (normalizedMime === "image/webp") return "webp";
|
||
if (normalizedMime === "image/gif") return "gif";
|
||
if (normalizedMime === "video/mp4") return "mp4";
|
||
if (normalizedMime === "video/webm") return "webm";
|
||
if (normalizedMime === "video/quicktime") return "mov";
|
||
|
||
try {
|
||
const matched = new URL(url).pathname.match(/\.([a-z0-9]{2,5})$/i);
|
||
if (matched?.[1]) return matched[1].toLowerCase();
|
||
} catch {
|
||
// Keep mime fallback below.
|
||
}
|
||
|
||
return mimeType.startsWith("video/") ? "mp4" : "png";
|
||
}
|
||
|
||
function buildDurableMediaName(prefix: string, url: string, mimeType: string): string {
|
||
const normalized = prefix
|
||
.trim()
|
||
.replace(/[\\/:*?"<>|]+/g, "-")
|
||
.replace(/\s+/g, " ")
|
||
.slice(0, 80)
|
||
.trim();
|
||
return `${normalized || "ecommerce-video-media"}.${getMediaExtension(url, mimeType)}`;
|
||
}
|
||
|
||
export async function resolveDurableMediaUrl(
|
||
url: string | null | undefined,
|
||
options: {
|
||
mediaType: "image" | "video";
|
||
namePrefix: string;
|
||
scope?: string;
|
||
uploadAssetByUrl?: UploadAssetByUrl;
|
||
},
|
||
): Promise<DurableMediaUrl> {
|
||
const sourceUrl = String(url || "").trim();
|
||
if (!sourceUrl) return { url: null };
|
||
if (isDurableOssUrl(sourceUrl)) return { url: sourceUrl };
|
||
|
||
const mimeType = options.mediaType === "video" ? "video/mp4" : "image/png";
|
||
const uploadAssetByUrl = options.uploadAssetByUrl || aiGenerationClient.uploadAssetByUrl.bind(aiGenerationClient);
|
||
|
||
try {
|
||
const uploaded = await uploadAssetByUrl({
|
||
sourceUrl,
|
||
name: buildDurableMediaName(options.namePrefix, sourceUrl, mimeType),
|
||
mimeType,
|
||
scope: options.scope || ecommerceOssScopes.videoHistory,
|
||
});
|
||
return {
|
||
url: uploaded.url || null,
|
||
originalUrl: sourceUrl,
|
||
ossKey: uploaded.ossKey || null,
|
||
};
|
||
} catch (error) {
|
||
const message = error instanceof Error ? error.message : String(error || "");
|
||
console.warn("[ecommerce-video] history media persistence failed:", message);
|
||
if (isTemporaryProviderUrl(sourceUrl)) {
|
||
return { url: null, originalUrl: sourceUrl };
|
||
}
|
||
return { url: sourceUrl };
|
||
}
|
||
}
|
||
|
||
export interface PlanCallbacks {
|
||
onStepStart: (step: PlanStep) => void;
|
||
onStepDone: (step: PlanStep) => void;
|
||
onImagesUploaded?: (urls: string[]) => void;
|
||
onUploadRejected?: (messages: string[]) => void;
|
||
onPartialProgress?: (progress: EcommerceVideoPlanProgress) => void;
|
||
signal?: AbortSignal;
|
||
/** Partial state from a previous run; steps with existing data are skipped. */
|
||
resumeFrom?: EcommerceVideoPlanProgress;
|
||
}
|
||
|
||
const LOCAL_PREVIEW_MISSING_FILE_MESSAGE = "Please re-upload the product image before generating the short video.";
|
||
|
||
export function buildComplianceFailureMessage(compliance: NonNullable<EcommerceVideoPlanProgress["compliance"]>): string {
|
||
const issues = compliance.issues
|
||
.slice(0, 3)
|
||
.map((issue) => [issue.field, issue.problem, issue.suggestion].filter(Boolean).join("："))
|
||
.filter(Boolean)
|
||
.join("；");
|
||
|
||
return issues
|
||
? `合规检查未通过，已停止生成。${issues}`
|
||
: "合规检查未通过，已停止生成。请修改商品说明或广告文案后重试。";
|
||
}
|
||
|
||
function readBlobAsDataUrl(blob: Blob): Promise<string> {
|
||
return new Promise((resolve, reject) => {
|
||
const reader = new FileReader();
|
||
reader.onload = () => resolve(String(reader.result || ""));
|
||
reader.onerror = () => reject(reader.error || new Error("File read failed"));
|
||
reader.readAsDataURL(blob);
|
||
});
|
||
}
|
||
|
||
function normalizeRemoteImageUrl(source: string): string | null {
|
||
try {
|
||
const url = new URL(source, typeof window !== "undefined" ? window.location.href : undefined);
|
||
return url.protocol === "http:" || url.protocol === "https:" ? url.href : null;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function uploadProductImageSource(source: string | Blob): Promise<string> {
|
||
if (typeof source === "string") {
|
||
if (source.startsWith("blob:")) {
|
||
throw new Error(LOCAL_PREVIEW_MISSING_FILE_MESSAGE);
|
||
}
|
||
|
||
if (source.startsWith("data:")) {
|
||
const mimeType = normalizeEcommerceImageMime(source.match(/^data:([^;,]+)/)?.[1] || "image/png");
|
||
const result = await aiGenerationClient.uploadAsset({ dataUrl: source, mimeType, scope: ecommerceOssScopes.videoSource });
|
||
return result.url;
|
||
}
|
||
|
||
const remoteUrl = normalizeRemoteImageUrl(source);
|
||
if (remoteUrl) {
|
||
const result = await aiGenerationClient.uploadAssetByUrl({ sourceUrl: remoteUrl, scope: ecommerceOssScopes.videoSource });
|
||
return result.url;
|
||
}
|
||
|
||
throw new Error("Unsupported product image URL. Please re-upload the product image.");
|
||
}
|
||
|
||
const mimeType = normalizeEcommerceImageMime(source.type || "image/png");
|
||
const blob = source.type === mimeType ? source : new Blob([source], { type: mimeType });
|
||
const dataUrl = await readBlobAsDataUrl(blob);
|
||
const result = await aiGenerationClient.uploadAsset({ dataUrl, mimeType, scope: ecommerceOssScopes.videoSource });
|
||
return result.url;
|
||
}
|
||
|
||
/**
|
||
* Run the full ad video planning pipeline.
|
||
* Supports resumption: if `resumeFrom` contains data for a step, that step is skipped.
|
||
* After each step, `onPartialProgress` fires so callers can persist intermediate state.
|
||
*/
|
||
export async function runVideoPlan(
|
||
imageSources: Array<string | Blob>,
|
||
manualText: string,
|
||
config: AdVideoUserConfig,
|
||
callbacks: PlanCallbacks,
|
||
): Promise<EcommerceVideoPlanResult> {
|
||
const { onStepStart, onStepDone, signal, resumeFrom = {} } = callbacks;
|
||
const progress: EcommerceVideoPlanProgress = { ...resumeFrom };
|
||
const emit = () => callbacks.onPartialProgress?.({ ...progress });
|
||
|
||
// Step: upload
|
||
if (!progress.imageUrls?.length) {
|
||
onStepStart("upload");
|
||
const imageUrls: string[] = [];
|
||
const rejected: string[] = [];
|
||
for (const source of imageSources) {
|
||
try {
|
||
imageUrls.push(await uploadProductImageSource(source));
|
||
} catch (err) {
|
||
rejected.push(err instanceof Error ? err.message : "Image upload failed");
|
||
}
|
||
}
|
||
if (rejected.length) {
|
||
progress.uploadWarnings = rejected;
|
||
callbacks.onUploadRejected?.(rejected);
|
||
}
|
||
if (!imageUrls.length) throw new Error("Image upload failed. Please check the image format or network and try again.");
|
||
progress.imageUrls = imageUrls;
|
||
onStepDone("upload");
|
||
callbacks.onImagesUploaded?.(imageUrls);
|
||
emit();
|
||
}
|
||
|
||
// Step: analyze
|
||
if (progress.imageDescription === undefined) {
|
||
onStepStart("analyze");
|
||
progress.imageDescription = await analyzeProductImages(progress.imageUrls!, signal);
|
||
onStepDone("analyze");
|
||
emit();
|
||
}
|
||
|
||
// Step: summary
|
||
if (!progress.summary) {
|
||
onStepStart("summary");
|
||
progress.summary = await buildProductSummary(progress.imageDescription || "", manualText, signal);
|
||
onStepDone("summary");
|
||
emit();
|
||
}
|
||
|
||
// Step: selling
|
||
if (!progress.selling) {
|
||
onStepStart("selling");
|
||
progress.selling = await extractSellingPoints(progress.summary, signal);
|
||
onStepDone("selling");
|
||
emit();
|
||
}
|
||
|
||
// Step: creative
|
||
if (!progress.creatives?.length) {
|
||
onStepStart("creative");
|
||
progress.creatives = await generateCreativeOptions(progress.selling, config, signal);
|
||
if (!progress.creatives.length) throw new Error("Failed to generate valid ad creatives.");
|
||
onStepDone("creative");
|
||
emit();
|
||
}
|
||
|
||
// Step: storyboard
|
||
if (!progress.storyboard) {
|
||
onStepStart("storyboard");
|
||
progress.storyboard = await generateStoryboard(progress.creatives[0], progress.summary, config, signal);
|
||
onStepDone("storyboard");
|
||
emit();
|
||
}
|
||
|
||
// Step: prompts
|
||
if (!progress.videoPrompts) {
|
||
onStepStart("prompts");
|
||
progress.videoPrompts = await generateVideoPrompts(progress.storyboard, progress.summary, signal);
|
||
onStepDone("prompts");
|
||
emit();
|
||
}
|
||
|
||
// Step: compliance
|
||
if (!progress.compliance) {
|
||
onStepStart("compliance");
|
||
progress.compliance = await checkCompliance(progress.summary, progress.selling, progress.storyboard, signal);
|
||
onStepDone("compliance");
|
||
emit();
|
||
}
|
||
|
||
if (progress.compliance.allow_video_generation === false) {
|
||
throw new Error(buildComplianceFailureMessage(progress.compliance));
|
||
}
|
||
|
||
return {
|
||
imageUrls: progress.imageUrls!,
|
||
imageDescription: progress.imageDescription,
|
||
summary: progress.summary!,
|
||
selling: progress.selling!,
|
||
creatives: progress.creatives!,
|
||
storyboard: progress.storyboard!,
|
||
videoPrompts: progress.videoPrompts!,
|
||
compliance: progress.compliance!,
|
||
};
|
||
}
|
||
|
||
export interface RenderSceneImageInput {
|
||
sceneId: number;
|
||
prompt: string;
|
||
aspectRatio: string;
|
||
productImageUrls: string[];
|
||
}
|
||
|
||
export interface RenderImageCallbacks {
|
||
onSceneImageSubmitted: (sceneId: number, taskId: string) => void;
|
||
onSceneImageProgress: (sceneId: number, progress: number) => void;
|
||
onSceneImageCompleted: (sceneId: number, resultUrl: string) => void;
|
||
onSceneImageFailed: (sceneId: number, error: string) => void;
|
||
}
|
||
|
||
export async function renderSceneImage(
|
||
input: RenderSceneImageInput,
|
||
callbacks: RenderImageCallbacks,
|
||
abortRef: { current: boolean },
|
||
): Promise<void> {
|
||
const { taskId } = await aiGenerationClient.createImageTask({
|
||
prompt: input.prompt,
|
||
ratio: input.aspectRatio,
|
||
quality: "2K",
|
||
referenceUrls: input.productImageUrls,
|
||
});
|
||
|
||
callbacks.onSceneImageSubmitted(input.sceneId, taskId);
|
||
|
||
const resultUrl = await waitForTask(taskId, {
|
||
abortRef,
|
||
kind: "image",
|
||
onProgress: (e) => callbacks.onSceneImageProgress(input.sceneId, e.progress),
|
||
});
|
||
|
||
if (resultUrl) {
|
||
callbacks.onSceneImageCompleted(input.sceneId, resultUrl);
|
||
} else {
|
||
callbacks.onSceneImageFailed(input.sceneId, "Image generation returned no result.");
|
||
}
|
||
}
|
||
|
||
export interface RenderSceneInput {
|
||
sceneId: number;
|
||
prompt: string;
|
||
durationSeconds: number;
|
||
imageUrl: string;
|
||
productImageUrls: string[];
|
||
aspectRatio: string;
|
||
resolution: string;
|
||
model?: string;
|
||
}
|
||
|
||
export interface RenderCallbacks {
|
||
onSceneSubmitted: (sceneId: number, taskId: string) => void;
|
||
onSceneProgress: (sceneId: number, progress: number) => void;
|
||
onSceneCompleted: (sceneId: number, resultUrl: string) => void;
|
||
onSceneFailed: (sceneId: number, error: string) => void;
|
||
}
|
||
|
||
export async function renderScene(
|
||
input: RenderSceneInput,
|
||
callbacks: RenderCallbacks,
|
||
abortRef: { current: boolean },
|
||
): Promise<void> {
|
||
const allReferenceUrls = [...input.productImageUrls, input.imageUrl];
|
||
const model = resolveVideoRequestModel({
|
||
model: input.model || ENTERPRISE_DEFAULT_VIDEO_MODEL,
|
||
referenceUrls: allReferenceUrls,
|
||
});
|
||
|
||
const { taskId } = await aiGenerationClient.createVideoTask({
|
||
model,
|
||
prompt: input.prompt,
|
||
ratio: input.aspectRatio,
|
||
duration: input.durationSeconds,
|
||
quality: input.resolution,
|
||
resolution: input.resolution,
|
||
frameMode: "start-end",
|
||
referenceUrls: allReferenceUrls,
|
||
hasReferenceVideo: false,
|
||
});
|
||
|
||
callbacks.onSceneSubmitted(input.sceneId, taskId);
|
||
|
||
const resultUrl = await waitForTask(taskId, {
|
||
abortRef,
|
||
kind: "video",
|
||
model,
|
||
onProgress: (e) => callbacks.onSceneProgress(input.sceneId, e.progress),
|
||
});
|
||
|
||
if (resultUrl) {
|
||
callbacks.onSceneCompleted(input.sceneId, resultUrl);
|
||
} else {
|
||
callbacks.onSceneFailed(input.sceneId, "Task returned no result.");
|
||
}
|
||
}
|
||
|
||
export function buildSceneTasks(
|
||
plan: EcommerceVideoPlanResult,
|
||
): EcommerceVideoSceneTask[] {
|
||
return plan.storyboard.scenes.map((scene) => {
|
||
const matchedPrompt = plan.videoPrompts.find((p) => p.scene_id === scene.scene_id);
|
||
return {
|
||
sceneId: scene.scene_id,
|
||
prompt: matchedPrompt?.positive_prompt || scene.visual_description,
|
||
durationSeconds: Number.parseInt(scene.duration, 10) || 5,
|
||
status: "idle" as const,
|
||
progress: 0,
|
||
};
|
||
});
|
||
}
|
||
|
||
// Video History API
|
||
|
||
export interface VideoHistoryScene {
|
||
sceneId: number;
|
||
prompt: string;
|
||
imageUrl?: string | null;
|
||
videoUrl?: string | null;
|
||
}
|
||
|
||
interface SaveVideoHistoryPayload {
|
||
title: string;
|
||
config: Record<string, unknown>;
|
||
plan: Record<string, unknown>;
|
||
scenes: VideoHistoryScene[];
|
||
sourceImageUrls: string[];
|
||
uploadAssetByUrl?: UploadAssetByUrl;
|
||
}
|
||
|
||
export interface VideoHistoryItem {
|
||
id: number;
|
||
title: string;
|
||
config: Record<string, unknown>;
|
||
scenes: VideoHistoryScene[];
|
||
sourceImageUrls: string[];
|
||
createdAt: string;
|
||
}
|
||
|
||
export interface VideoHistoryListResponse {
|
||
items: VideoHistoryItem[];
|
||
total: number;
|
||
limit: number;
|
||
offset: number;
|
||
}
|
||
|
||
export async function buildDurableVideoHistoryPayload(payload: SaveVideoHistoryPayload): Promise<SaveVideoHistoryPayload> {
|
||
const uploadAssetByUrl = payload.uploadAssetByUrl;
|
||
const scenes = await Promise.all(
|
||
payload.scenes.map(async (scene) => {
|
||
const [image, video] = await Promise.all([
|
||
resolveDurableMediaUrl(scene.imageUrl, {
|
||
mediaType: "image",
|
||
namePrefix: `ecommerce-scene-${scene.sceneId}-image`,
|
||
uploadAssetByUrl,
|
||
}),
|
||
resolveDurableMediaUrl(scene.videoUrl, {
|
||
mediaType: "video",
|
||
namePrefix: `ecommerce-scene-${scene.sceneId}-video`,
|
||
uploadAssetByUrl,
|
||
}),
|
||
]);
|
||
return {
|
||
...scene,
|
||
imageUrl: image.url,
|
||
videoUrl: video.url,
|
||
};
|
||
}),
|
||
);
|
||
|
||
const sourceImageUrls = (
|
||
await Promise.all(
|
||
payload.sourceImageUrls.map((url, index) =>
|
||
resolveDurableMediaUrl(url, {
|
||
mediaType: "image",
|
||
namePrefix: `ecommerce-source-${index + 1}`,
|
||
uploadAssetByUrl,
|
||
}),
|
||
),
|
||
)
|
||
)
|
||
.map((item) => item.url)
|
||
.filter((url): url is string => Boolean(url));
|
||
|
||
return {
|
||
...payload,
|
||
scenes,
|
||
sourceImageUrls,
|
||
};
|
||
}
|
||
|
||
export async function saveVideoHistory(payload: SaveVideoHistoryPayload): Promise<{ id: number; createdAt: string }> {
|
||
const { uploadAssetByUrl: _uploadAssetByUrl, ...historyPayload } = await buildDurableVideoHistoryPayload(payload);
|
||
return serverRequest<{ id: number; createdAt: string }>("ai/ecommerce/video-history", {
|
||
method: "POST",
|
||
body: historyPayload,
|
||
maxRetries: 0,
|
||
fallbackMessage: "Failed to save video history",
|
||
});
|
||
}
|
||
|
||
function removeTemporaryHistoryUrls(item: VideoHistoryItem): VideoHistoryItem {
|
||
return {
|
||
...item,
|
||
scenes: item.scenes.map((scene) => ({
|
||
...scene,
|
||
imageUrl: scene.imageUrl && !isTemporaryProviderUrl(scene.imageUrl) ? scene.imageUrl : null,
|
||
videoUrl: scene.videoUrl && !isTemporaryProviderUrl(scene.videoUrl) ? scene.videoUrl : null,
|
||
})),
|
||
sourceImageUrls: item.sourceImageUrls.filter((url) => !isTemporaryProviderUrl(url)),
|
||
};
|
||
}
|
||
|
||
export async function fetchVideoHistory(
|
||
limit = 20,
|
||
offset = 0,
|
||
): Promise<VideoHistoryListResponse> {
|
||
const search = new URLSearchParams({ limit: String(limit), offset: String(offset) });
|
||
const history = await serverRequest<VideoHistoryListResponse>(`ai/ecommerce/video-history?${search}`, {
|
||
fallbackMessage: "Failed to fetch video history",
|
||
});
|
||
return {
|
||
...history,
|
||
items: history.items.map(removeTemporaryHistoryUrls),
|
||
};
|
||
}
|
||
|
||
export async function deleteVideoHistory(id: number): Promise<void> {
|
||
await serverRequest<void>(`ai/ecommerce/video-history/${id}`, {
|
||
method: "DELETE",
|
||
maxRetries: 0,
|
||
fallbackMessage: "Failed to delete video history",
|
||
});
|
||
}
|