import { analyzeProductImages, buildProductSummary, extractSellingPoints, generateCreativeOptions, generateStoryboard, generateVideoPrompts, checkCompliance, type AdVideoUserConfig, } from "../../api/adVideoPlanClient"; import { aiGenerationClient } from "../../api/aiGenerationClient"; import { serverRequest } from "../../api/serverConnection"; import { waitForTask } from "../../api/taskSubscription"; import { ENTERPRISE_DEFAULT_VIDEO_MODEL } from "../../utils/enterpriseVideoPolicy"; import { resolveVideoRequestModel } from "../../utils/resolveVideoModel"; import { ecommerceOssScopes } from "./ecommerceGenerationPersistence"; import { normalizeEcommerceImageMime } from "./ecommerceImageValidation"; import type { EcommerceVideoPlanProgress, EcommerceVideoPlanResult, EcommerceVideoSceneTask, PlanStep, } from "./ecommerceVideoTypes"; type UploadAssetByUrl = typeof aiGenerationClient.uploadAssetByUrl; interface DurableMediaUrl { url: string | null; originalUrl?: string | null; ossKey?: string | null; } const TEMP_MEDIA_HOST_RE = /^file\d*\.aitohumanize\.com$/i; const OSS_MEDIA_HOST_RE = /\.oss-[^.]+\.aliyuncs\.com$/i; function isTemporaryProviderUrl(url: string): boolean { try { return TEMP_MEDIA_HOST_RE.test(new URL(url).hostname); } catch { return false; } } function isDurableOssUrl(url: string): boolean { try { const parsed = new URL(url); return parsed.protocol === "https:" && OSS_MEDIA_HOST_RE.test(parsed.hostname); } catch { return false; } } function getMediaExtension(url: string, mimeType: string): string { const normalizedMime = mimeType.split(";")[0]?.trim().toLowerCase(); if (normalizedMime === "image/jpeg") return "jpg"; if (normalizedMime === "image/png") return "png"; if (normalizedMime === "image/webp") return "webp"; if (normalizedMime === "image/gif") return "gif"; if (normalizedMime === "video/mp4") return "mp4"; if (normalizedMime === "video/webm") return "webm"; if (normalizedMime === "video/quicktime") return "mov"; try { const matched = new URL(url).pathname.match(/\.([a-z0-9]{2,5})$/i); if (matched?.[1]) return matched[1].toLowerCase(); } catch { // Keep mime fallback below. } return mimeType.startsWith("video/") ? "mp4" : "png"; } function buildDurableMediaName(prefix: string, url: string, mimeType: string): string { const normalized = prefix .trim() .replace(/[\\/:*?"<>|]+/g, "-") .replace(/\s+/g, " ") .slice(0, 80) .trim(); return `${normalized || "ecommerce-video-media"}.${getMediaExtension(url, mimeType)}`; } export async function resolveDurableMediaUrl( url: string | null | undefined, options: { mediaType: "image" | "video"; namePrefix: string; scope?: string; uploadAssetByUrl?: UploadAssetByUrl; }, ): Promise { const sourceUrl = String(url || "").trim(); if (!sourceUrl) return { url: null }; if (isDurableOssUrl(sourceUrl)) return { url: sourceUrl }; const mimeType = options.mediaType === "video" ? "video/mp4" : "image/png"; const uploadAssetByUrl = options.uploadAssetByUrl || aiGenerationClient.uploadAssetByUrl.bind(aiGenerationClient); try { const uploaded = await uploadAssetByUrl({ sourceUrl, name: buildDurableMediaName(options.namePrefix, sourceUrl, mimeType), mimeType, scope: options.scope || ecommerceOssScopes.videoHistory, }); return { url: uploaded.url || null, originalUrl: sourceUrl, ossKey: uploaded.ossKey || null, }; } catch (error) { const message = error instanceof Error ? error.message : String(error || ""); console.warn("[ecommerce-video] history media persistence failed:", message); if (isTemporaryProviderUrl(sourceUrl)) { return { url: null, originalUrl: sourceUrl }; } return { url: sourceUrl }; } } export interface PlanCallbacks { onStepStart: (step: PlanStep) => void; onStepDone: (step: PlanStep) => void; onImagesUploaded?: (urls: string[]) => void; onUploadRejected?: (messages: string[]) => void; onPartialProgress?: (progress: EcommerceVideoPlanProgress) => void; signal?: AbortSignal; /** Partial state from a previous run; steps with existing data are skipped. */ resumeFrom?: EcommerceVideoPlanProgress; } const LOCAL_PREVIEW_MISSING_FILE_MESSAGE = "Please re-upload the product image before generating the short video."; export function buildComplianceFailureMessage(compliance: NonNullable): string { const issues = compliance.issues .slice(0, 3) .map((issue) => [issue.field, issue.problem, issue.suggestion].filter(Boolean).join(":")) .filter(Boolean) .join(";"); return issues ? `合规检查未通过,已停止生成。${issues}` : "合规检查未通过,已停止生成。请修改商品说明或广告文案后重试。"; } function readBlobAsDataUrl(blob: Blob): Promise { return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(String(reader.result || "")); reader.onerror = () => reject(reader.error || new Error("File read failed")); reader.readAsDataURL(blob); }); } function normalizeRemoteImageUrl(source: string): string | null { try { const url = new URL(source, typeof window !== "undefined" ? window.location.href : undefined); return url.protocol === "http:" || url.protocol === "https:" ? url.href : null; } catch { return null; } } async function uploadProductImageSource(source: string | Blob): Promise { if (typeof source === "string") { if (source.startsWith("blob:")) { throw new Error(LOCAL_PREVIEW_MISSING_FILE_MESSAGE); } if (source.startsWith("data:")) { const mimeType = normalizeEcommerceImageMime(source.match(/^data:([^;,]+)/)?.[1] || "image/png"); const result = await aiGenerationClient.uploadAsset({ dataUrl: source, mimeType, scope: ecommerceOssScopes.videoSource }); return result.url; } const remoteUrl = normalizeRemoteImageUrl(source); if (remoteUrl) { const result = await aiGenerationClient.uploadAssetByUrl({ sourceUrl: remoteUrl, scope: ecommerceOssScopes.videoSource }); return result.url; } throw new Error("Unsupported product image URL. Please re-upload the product image."); } const mimeType = normalizeEcommerceImageMime(source.type || "image/png"); const blob = source.type === mimeType ? source : new Blob([source], { type: mimeType }); const dataUrl = await readBlobAsDataUrl(blob); const result = await aiGenerationClient.uploadAsset({ dataUrl, mimeType, scope: ecommerceOssScopes.videoSource }); return result.url; } /** * Run the full ad video planning pipeline. * Supports resumption: if `resumeFrom` contains data for a step, that step is skipped. * After each step, `onPartialProgress` fires so callers can persist intermediate state. */ export async function runVideoPlan( imageSources: Array, manualText: string, config: AdVideoUserConfig, callbacks: PlanCallbacks, ): Promise { const { onStepStart, onStepDone, signal, resumeFrom = {} } = callbacks; const progress: EcommerceVideoPlanProgress = { ...resumeFrom }; const emit = () => callbacks.onPartialProgress?.({ ...progress }); // Step: upload if (!progress.imageUrls?.length) { onStepStart("upload"); const imageUrls: string[] = []; const rejected: string[] = []; for (const source of imageSources) { try { imageUrls.push(await uploadProductImageSource(source)); } catch (err) { rejected.push(err instanceof Error ? err.message : "Image upload failed"); } } if (rejected.length) { progress.uploadWarnings = rejected; callbacks.onUploadRejected?.(rejected); } if (!imageUrls.length) throw new Error("Image upload failed. Please check the image format or network and try again."); progress.imageUrls = imageUrls; onStepDone("upload"); callbacks.onImagesUploaded?.(imageUrls); emit(); } // Step: analyze if (progress.imageDescription === undefined) { onStepStart("analyze"); progress.imageDescription = await analyzeProductImages(progress.imageUrls!, signal); onStepDone("analyze"); emit(); } // Step: summary if (!progress.summary) { onStepStart("summary"); progress.summary = await buildProductSummary(progress.imageDescription || "", manualText, signal); onStepDone("summary"); emit(); } // Step: selling if (!progress.selling) { onStepStart("selling"); progress.selling = await extractSellingPoints(progress.summary, signal); onStepDone("selling"); emit(); } // Step: creative if (!progress.creatives?.length) { onStepStart("creative"); progress.creatives = await generateCreativeOptions(progress.selling, config, signal); if (!progress.creatives.length) throw new Error("Failed to generate valid ad creatives."); onStepDone("creative"); emit(); } // Step: storyboard if (!progress.storyboard) { onStepStart("storyboard"); progress.storyboard = await generateStoryboard(progress.creatives[0], progress.summary, config, signal); onStepDone("storyboard"); emit(); } // Step: prompts if (!progress.videoPrompts) { onStepStart("prompts"); progress.videoPrompts = await generateVideoPrompts(progress.storyboard, progress.summary, signal); onStepDone("prompts"); emit(); } // Step: compliance if (!progress.compliance) { onStepStart("compliance"); progress.compliance = await checkCompliance(progress.summary, progress.selling, progress.storyboard, signal); onStepDone("compliance"); emit(); } if (progress.compliance.allow_video_generation === false) { throw new Error(buildComplianceFailureMessage(progress.compliance)); } return { imageUrls: progress.imageUrls!, imageDescription: progress.imageDescription, summary: progress.summary!, selling: progress.selling!, creatives: progress.creatives!, storyboard: progress.storyboard!, videoPrompts: progress.videoPrompts!, compliance: progress.compliance!, }; } export interface RenderSceneImageInput { sceneId: number; prompt: string; aspectRatio: string; productImageUrls: string[]; } export interface RenderImageCallbacks { onSceneImageSubmitted: (sceneId: number, taskId: string) => void; onSceneImageProgress: (sceneId: number, progress: number) => void; onSceneImageCompleted: (sceneId: number, resultUrl: string) => void; onSceneImageFailed: (sceneId: number, error: string) => void; } export async function renderSceneImage( input: RenderSceneImageInput, callbacks: RenderImageCallbacks, abortRef: { current: boolean }, ): Promise { const { taskId } = await aiGenerationClient.createImageTask({ prompt: input.prompt, ratio: input.aspectRatio, quality: "2K", referenceUrls: input.productImageUrls, }); callbacks.onSceneImageSubmitted(input.sceneId, taskId); const resultUrl = await waitForTask(taskId, { abortRef, kind: "image", onProgress: (e) => callbacks.onSceneImageProgress(input.sceneId, e.progress), }); if (resultUrl) { callbacks.onSceneImageCompleted(input.sceneId, resultUrl); } else { callbacks.onSceneImageFailed(input.sceneId, "Image generation returned no result."); } } export interface RenderSceneInput { sceneId: number; prompt: string; durationSeconds: number; imageUrl: string; productImageUrls: string[]; aspectRatio: string; resolution: string; model?: string; } export interface RenderCallbacks { onSceneSubmitted: (sceneId: number, taskId: string) => void; onSceneProgress: (sceneId: number, progress: number) => void; onSceneCompleted: (sceneId: number, resultUrl: string) => void; onSceneFailed: (sceneId: number, error: string) => void; } export async function renderScene( input: RenderSceneInput, callbacks: RenderCallbacks, abortRef: { current: boolean }, ): Promise { const allReferenceUrls = [...input.productImageUrls, input.imageUrl]; const model = resolveVideoRequestModel({ model: input.model || ENTERPRISE_DEFAULT_VIDEO_MODEL, referenceUrls: allReferenceUrls, }); const { taskId } = await aiGenerationClient.createVideoTask({ model, prompt: input.prompt, ratio: input.aspectRatio, duration: input.durationSeconds, quality: input.resolution, resolution: input.resolution, frameMode: "start-end", referenceUrls: allReferenceUrls, hasReferenceVideo: false, }); callbacks.onSceneSubmitted(input.sceneId, taskId); const resultUrl = await waitForTask(taskId, { abortRef, kind: "video", model, onProgress: (e) => callbacks.onSceneProgress(input.sceneId, e.progress), }); if (resultUrl) { callbacks.onSceneCompleted(input.sceneId, resultUrl); } else { callbacks.onSceneFailed(input.sceneId, "Task returned no result."); } } export function buildSceneTasks( plan: EcommerceVideoPlanResult, ): EcommerceVideoSceneTask[] { return plan.storyboard.scenes.map((scene) => { const matchedPrompt = plan.videoPrompts.find((p) => p.scene_id === scene.scene_id); return { sceneId: scene.scene_id, prompt: matchedPrompt?.positive_prompt || scene.visual_description, durationSeconds: Number.parseInt(scene.duration, 10) || 5, status: "idle" as const, progress: 0, }; }); } // Video History API export interface VideoHistoryScene { sceneId: number; prompt: string; imageUrl?: string | null; videoUrl?: string | null; } interface SaveVideoHistoryPayload { title: string; config: Record; plan: Record; scenes: VideoHistoryScene[]; sourceImageUrls: string[]; uploadAssetByUrl?: UploadAssetByUrl; } export interface VideoHistoryItem { id: number; title: string; config: Record; scenes: VideoHistoryScene[]; sourceImageUrls: string[]; createdAt: string; } export interface VideoHistoryListResponse { items: VideoHistoryItem[]; total: number; limit: number; offset: number; } export async function buildDurableVideoHistoryPayload(payload: SaveVideoHistoryPayload): Promise { const uploadAssetByUrl = payload.uploadAssetByUrl; const scenes = await Promise.all( payload.scenes.map(async (scene) => { const [image, video] = await Promise.all([ resolveDurableMediaUrl(scene.imageUrl, { mediaType: "image", namePrefix: `ecommerce-scene-${scene.sceneId}-image`, uploadAssetByUrl, }), resolveDurableMediaUrl(scene.videoUrl, { mediaType: "video", namePrefix: `ecommerce-scene-${scene.sceneId}-video`, uploadAssetByUrl, }), ]); return { ...scene, imageUrl: image.url, videoUrl: video.url, }; }), ); const sourceImageUrls = ( await Promise.all( payload.sourceImageUrls.map((url, index) => resolveDurableMediaUrl(url, { mediaType: "image", namePrefix: `ecommerce-source-${index + 1}`, uploadAssetByUrl, }), ), ) ) .map((item) => item.url) .filter((url): url is string => Boolean(url)); return { ...payload, scenes, sourceImageUrls, }; } export async function saveVideoHistory(payload: SaveVideoHistoryPayload): Promise<{ id: number; createdAt: string }> { const { uploadAssetByUrl: _uploadAssetByUrl, ...historyPayload } = await buildDurableVideoHistoryPayload(payload); return serverRequest<{ id: number; createdAt: string }>("ai/ecommerce/video-history", { method: "POST", body: historyPayload, maxRetries: 0, fallbackMessage: "Failed to save video history", }); } function removeTemporaryHistoryUrls(item: VideoHistoryItem): VideoHistoryItem { return { ...item, scenes: item.scenes.map((scene) => ({ ...scene, imageUrl: scene.imageUrl && !isTemporaryProviderUrl(scene.imageUrl) ? scene.imageUrl : null, videoUrl: scene.videoUrl && !isTemporaryProviderUrl(scene.videoUrl) ? scene.videoUrl : null, })), sourceImageUrls: item.sourceImageUrls.filter((url) => !isTemporaryProviderUrl(url)), }; } export async function fetchVideoHistory( limit = 20, offset = 0, ): Promise { const search = new URLSearchParams({ limit: String(limit), offset: String(offset) }); const history = await serverRequest(`ai/ecommerce/video-history?${search}`, { fallbackMessage: "Failed to fetch video history", }); return { ...history, items: history.items.map(removeTemporaryHistoryUrls), }; } export async function deleteVideoHistory(id: number): Promise { await serverRequest(`ai/ecommerce/video-history/${id}`, { method: "DELETE", maxRetries: 0, fallbackMessage: "Failed to delete video history", }); }