Files
omniai-ds-code-package/src/api/adVideoPlanClient.ts
T

385 lines
16 KiB
TypeScript
Raw Normal View History

2026-06-10 14:06:16 +08:00
import { buildApiUrl, buildAuthHeaders } from "./serverConnection";
type AbortSignalConstructorWithAny = typeof AbortSignal & {
any?: (signals: AbortSignal[]) => AbortSignal;
};
function combineAbortSignals(signal: AbortSignal | undefined, timeoutSignal: AbortSignal): AbortSignal {
if (!signal) return timeoutSignal;
const abortSignal = AbortSignal as AbortSignalConstructorWithAny;
if (typeof abortSignal.any === "function") return abortSignal.any([signal, timeoutSignal]);
const controller = new AbortController();
const abortFrom = (source: AbortSignal) => {
if (!controller.signal.aborted) controller.abort(source.reason);
};
if (signal.aborted) abortFrom(signal);
else signal.addEventListener("abort", () => abortFrom(signal), { once: true });
if (timeoutSignal.aborted) abortFrom(timeoutSignal);
else timeoutSignal.addEventListener("abort", () => abortFrom(timeoutSignal), { once: true });
return controller.signal;
}
2026-06-10 14:06:16 +08:00
export interface AdVideoUserConfig {
platform: string;
aspectRatio: string;
durationSeconds: number;
style: string;
language: string;
market: string;
needVoiceover: boolean;
needSubtitle: boolean;
conversionFocus: "conversion" | "brand";
}
export interface ProductSummary {
product_name: string;
category: string;
appearance: string;
materials: string[];
colors: string[];
core_features: string[];
target_users: string[];
usage_scenarios: string[];
selling_points: string[];
risk_notes: string[];
}
export interface SellingPoint {
point: string;
evidence: string;
ad_expression: string;
}
export interface SellingPointResult {
primary_selling_points: SellingPoint[];
secondary_selling_points: SellingPoint[];
unsupported_claims: string[];
compliance_warnings: string[];
}
export interface CreativeOption {
creative_id: string;
creative_type: string;
hook: string;
target_user: string;
main_message: string;
emotional_tone: string;
recommended_platform: string;
reason: string;
}
export interface StoryboardScene {
scene_id: number;
duration: string;
scene_goal: string;
visual_description: string;
product_focus: string;
camera_movement: string;
background: string;
lighting: string;
subtitle: string;
voiceover: string;
transition: string;
}
export interface Storyboard {
video_title: string;
duration: string;
aspect_ratio: string;
target_platform: string;
language: string;
scenes: StoryboardScene[];
}
export interface VideoPrompt {
scene_id: number;
positive_prompt: string;
negative_prompt: string;
reference_requirements: string;
consistency_rules: string;
text_overlay: string;
}
export interface ComplianceCheck {
risk_level: "low" | "medium" | "high";
issues: Array<{ field: string; problem: string; suggestion: string }>;
allow_video_generation: boolean;
}
2026-06-12 17:25:30 +08:00
function findJsonSlice(raw: string): string {
const start = raw.search(/[\[{]/);
if (start < 0) return raw;
const stack: string[] = [];
let inString = false;
let escaped = false;
for (let index = start; index < raw.length; index += 1) {
const char = raw[index];
if (inString) {
if (escaped) {
escaped = false;
} else if (char === "\\") {
escaped = true;
} else if (char === "\"") {
inString = false;
}
continue;
}
if (char === "\"") {
inString = true;
} else if (char === "{" || char === "[") {
stack.push(char === "{" ? "}" : "]");
} else if (char === "}" || char === "]") {
if (stack.pop() !== char) break;
if (stack.length === 0) return raw.slice(start, index + 1);
}
}
return raw.slice(start);
}
2026-06-10 14:06:16 +08:00
function extractJson(text: string): unknown {
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/);
const raw = fenced ? fenced[1].trim() : text.trim();
2026-06-12 17:25:30 +08:00
const slice = findJsonSlice(raw);
2026-06-10 14:06:16 +08:00
try {
return JSON.parse(slice);
} catch {
throw new Error("AI 返回内容不是有效的 JSON");
}
}
2026-06-12 17:25:30 +08:00
type ChatContent =
| string
| Array<
| { type: "image_url"; image_url: { url: string } }
| { type: "text"; text: string }
>;
2026-06-10 14:06:16 +08:00
interface ChatMessage {
role: "system" | "user";
2026-06-12 17:25:30 +08:00
content: ChatContent;
2026-06-10 14:06:16 +08:00
}
const MAX_RETRIES = 3;
const RETRY_BASE_MS = 2000;
const CHAT_TIMEOUT_MS = 180_000; // 3 minutes per AI call (server times out at 120s + network slack)
// 5xx, 429, network failures, timeouts, and AbortError-from-timeout are all retryable
function isTransientError(err: unknown): boolean {
if (!(err instanceof Error)) return false;
const msg = err.message.toLowerCase();
if (/\b(429|500|502|503|504|520|521|522|524)\b/.test(msg)) return true;
if (msg.includes("signal timed out") || msg.includes("timeout")) return true;
if (msg.includes("failed to fetch") || msg.includes("networkerror") || msg.includes("network error")) return true;
if (msg.includes("ai 调用失败") || msg.includes("图片理解调用失败")) return true; // generic upstream failures
return false;
}
async function retryOnTransient<T>(fn: () => Promise<T>, signal?: AbortSignal): Promise<T> {
let lastErr: unknown;
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
try {
return await fn();
} catch (err) {
lastErr = err;
if (signal?.aborted) throw err;
// External AbortError caused by our timeoutSignal — retryable
if (err instanceof Error && err.name === "AbortError" && !signal?.aborted) {
if (attempt === MAX_RETRIES) throw err;
const delay = RETRY_BASE_MS * 2 ** attempt + Math.random() * 1000;
await new Promise((r) => setTimeout(r, delay));
continue;
}
if (attempt === MAX_RETRIES) throw err;
if (!isTransientError(err)) throw err;
const delay = RETRY_BASE_MS * 2 ** attempt + Math.random() * 1000;
await new Promise((r) => setTimeout(r, delay));
}
}
throw lastErr instanceof Error ? lastErr : new Error("AI 调用失败:已重试多次");
}
async function chat(
systemPrompt: string,
userContent: string,
options?: { model?: string; signal?: AbortSignal },
): Promise<string> {
2026-06-12 17:25:30 +08:00
return retryOnTransient(async () => {
const messages: ChatMessage[] = [
{ role: "system", content: systemPrompt },
{ role: "user", content: userContent },
];
const timeoutSignal = AbortSignal.timeout(CHAT_TIMEOUT_MS);
const combinedSignal = combineAbortSignals(options?.signal, timeoutSignal);
const body: Record<string, unknown> = { messages, stream: false, temperature: 0.4 };
if (options?.model) body.model = options.model;
2026-06-10 14:06:16 +08:00
2026-06-12 17:25:30 +08:00
const res = await fetch(buildApiUrl("ai/chat"), {
method: "POST",
headers: buildAuthHeaders(),
body: JSON.stringify(body),
signal: combinedSignal,
});
if (!res.ok) {
const errBody = await res.text().catch(() => "");
throw new Error(`AI 调用失败 (${res.status})${errBody ? `: ${errBody.slice(0, 120)}` : ""}`);
2026-06-10 14:06:16 +08:00
}
2026-06-12 17:25:30 +08:00
const payload = await res.json();
const content: string =
payload?.content ?? payload?.choices?.[0]?.message?.content ?? payload?.text ?? "";
if (!content) throw new Error("模型未返回有效内容");
return content;
}, options?.signal);
2026-06-10 14:06:16 +08:00
}
async function visionChat(
systemPrompt: string,
text: string,
imageUrls: string[],
signal?: AbortSignal,
): Promise<string> {
2026-06-12 17:25:30 +08:00
const content: ChatContent = [
...imageUrls.map((url) => ({ type: "image_url" as const, image_url: { url } })),
2026-06-10 14:06:16 +08:00
{ type: "text", text },
];
const messages = [
{ role: "system", content: systemPrompt },
{ role: "user", content },
2026-06-12 17:25:30 +08:00
] satisfies ChatMessage[];
2026-06-10 14:06:16 +08:00
2026-06-12 17:25:30 +08:00
return retryOnTransient(async () => {
2026-06-10 14:06:16 +08:00
const timeoutSignal = AbortSignal.timeout(CHAT_TIMEOUT_MS);
const combinedSignal = combineAbortSignals(signal, timeoutSignal);
2026-06-12 17:25:30 +08:00
const res = await fetch(buildApiUrl("ai/chat"), {
method: "POST",
headers: buildAuthHeaders(),
body: JSON.stringify({ messages, stream: false, temperature: 0.3 }),
signal: combinedSignal,
});
if (!res.ok) {
const errBody = await res.text().catch(() => "");
if (errBody.includes("image format")) throw new Error("图片格式不受支持,请更换图片后重试");
throw new Error(`图片理解调用失败 (${res.status})${errBody ? `: ${errBody.slice(0, 120)}` : ""}`);
2026-06-10 14:06:16 +08:00
}
2026-06-12 17:25:30 +08:00
const payload = await res.json();
const result: string =
payload?.content ?? payload?.choices?.[0]?.message?.content ?? payload?.text ?? "";
if (!result) throw new Error("图片理解未返回有效内容");
return result;
}, signal);
2026-06-10 14:06:16 +08:00
}
const IMAGE_UNDERSTANDING_PROMPT = `你是电商产品图片分析专家。请分析用户提供的产品图片,识别产品主体、外观、颜色、材质、形状、尺寸感、品牌标识、关键部件、可视化卖点和适合展示的镜头角度。请用简洁的中文段落描述,不要编造图片中看不到的信息。`;
export async function analyzeProductImages(
imageUrls: string[],
signal?: AbortSignal,
): Promise<string> {
if (imageUrls.length === 0) return "";
return visionChat(IMAGE_UNDERSTANDING_PROMPT, "请分析这些产品图片的视觉特征。", imageUrls, signal);
}
const PRODUCT_SUMMARY_PROMPT = `你是商品信息理解专家。根据产品图片理解结果和说明书文本,输出结构化的商品信息。严格按以下 JSON 格式返回,不要任何额外解释:
{"product_name":"","category":"","appearance":"","materials":[],"colors":[],"core_features":[],"target_users":[],"usage_scenarios":[],"selling_points":[],"risk_notes":[]}
要求:只描述资料中真实存在的信息,不要编造说明书或图片中不存在的功能。risk_notes 列出可能涉及夸大、医疗功效、绝对化用语等风险点。`;
export async function buildProductSummary(
imageDescription: string,
manualText: string,
signal?: AbortSignal,
): Promise<ProductSummary> {
const userContent = `【产品图片理解结果】\n${imageDescription || "(无图片)"}\n\n【产品说明书/详情文本】\n${manualText || "(无文本)"}`;
const text = await chat(PRODUCT_SUMMARY_PROMPT, userContent, { signal });
return extractJson(text) as ProductSummary;
}
const SELLING_POINT_PROMPT = `你是电商卖点提炼专家。将商品信息拆分为不同层级卖点。严格按以下 JSON 格式返回,不要任何额外解释:
{"primary_selling_points":[{"point":"","evidence":"","ad_expression":""}],"secondary_selling_points":[{"point":"","evidence":"","ad_expression":""}],"unsupported_claims":[],"compliance_warnings":[]}
要求:每个卖点必须有来源依据(evidence),依据来自输入的商品信息。不得凭空增加功能。无依据的卖点放入 unsupported_claims。涉及夸大、医疗、绝对化用语的放入 compliance_warnings。`;
export async function extractSellingPoints(
summary: ProductSummary,
signal?: AbortSignal,
): Promise<SellingPointResult> {
const text = await chat(SELLING_POINT_PROMPT, `【商品结构化信息】\n${JSON.stringify(summary, null, 2)}`, { signal });
return extractJson(text) as SellingPointResult;
}
function configBlock(config: AdVideoUserConfig): string {
return `【用户配置】\n目标平台:${config.platform}\n视频比例:${config.aspectRatio}\n时长:${config.durationSeconds}\n广告风格:${config.style}\n语言:${config.language}\n目标市场:${config.market}\n旁白:${config.needVoiceover ? "需要" : "不需要"}\n字幕:${config.needSubtitle ? "需要" : "不需要"}\n侧重:${config.conversionFocus === "conversion" ? "强转化" : "品牌展示"}`;
}
const CREATIVE_PROMPT = `你是电商广告创意专家。根据商品卖点和用户配置,生成至少 3 个差异化的广告创意方向。严格按以下 JSON 格式返回,不要任何额外解释:
{"creative_options":[{"creative_id":"A","creative_type":"","hook":"","target_user":"","main_message":"","emotional_tone":"","recommended_platform":"","reason":""}]}
要求:每个方向围绕真实卖点,有清晰广告逻辑,方向之间有明显差异。`;
export async function generateCreativeOptions(
selling: SellingPointResult,
config: AdVideoUserConfig,
signal?: AbortSignal,
): Promise<CreativeOption[]> {
const userContent = `【卖点】\n${JSON.stringify(selling.primary_selling_points, null, 2)}\n\n${configBlock(config)}`;
const text = await chat(CREATIVE_PROMPT, userContent, { signal });
const parsed = extractJson(text) as { creative_options?: CreativeOption[] };
return Array.isArray(parsed.creative_options) ? parsed.creative_options : [];
}
const STORYBOARD_PROMPT = `你是电商短视频分镜师。根据选定的广告创意方向、商品信息和用户配置,输出结构化视频分镜。严格按以下 JSON 格式返回,不要任何额外解释:
{"video_title":"","duration":"","aspect_ratio":"","target_platform":"","language":"","scenes":[{"scene_id":1,"duration":"3s","scene_goal":"","visual_description":"","product_focus":"","camera_movement":"","background":"","lighting":"","subtitle":"","voiceover":"","transition":""}]}
要求:开头3秒有吸引点,中段展示核心卖点,结尾有行动号召。各镜头时长之和等于配置总时长。不要出现说明书中不存在的功能,不要设计视频模型难以稳定生成的复杂动作。`;
export async function generateStoryboard(
creative: CreativeOption,
summary: ProductSummary,
config: AdVideoUserConfig,
signal?: AbortSignal,
): Promise<Storyboard> {
const userContent = `【选定创意方向】\n${JSON.stringify(creative, null, 2)}\n\n【商品信息】\n${JSON.stringify(summary, null, 2)}\n\n${configBlock(config)}`;
const text = await chat(STORYBOARD_PROMPT, userContent, { signal });
return extractJson(text) as Storyboard;
}
const VIDEO_PROMPT_PROMPT = `你是 AI 视频模型提示词工程师。为每个分镜生成视频模型提示词。严格按以下 JSON 格式返回一个数组,不要任何额外解释:
[{"scene_id":1,"positive_prompt":"","negative_prompt":"","reference_requirements":"","consistency_rules":"","text_overlay":""}]
正向提示词需包含:产品主体、外观、颜色、材质、使用场景、镜头构图、镜头运动、光线风格、背景环境、广告质感、画面节奏。
负面提示词需包含:不改变产品外观/颜色、不添加不存在的部件、不生成错误Logo、不生成模糊文字、不生成虚假功能演示、不生成畸形手部、不生成夸张功效、不生成医学暗示。
字幕和文字建议后期叠加(text_overlay),不要让视频模型直接生成文字。`;
export async function generateVideoPrompts(
storyboard: Storyboard,
summary: ProductSummary,
signal?: AbortSignal,
): Promise<VideoPrompt[]> {
const userContent = `【分镜脚本】\n${JSON.stringify(storyboard.scenes, null, 2)}\n\n【产品外观特征(一致性参考)】\n外观:${summary.appearance}\n颜色:${summary.colors.join("、")}\n材质:${summary.materials.join("、")}`;
const text = await chat(VIDEO_PROMPT_PROMPT, userContent, { signal });
const parsed = extractJson(text);
return Array.isArray(parsed) ? (parsed as VideoPrompt[]) : [];
}
const COMPLIANCE_PROMPT = `你是电商广告合规质检专家。检查文案和卖点是否存在虚假宣传、绝对化用语(如"最""第一""100%")、医疗功效暗示、高风险品类违规表达。严格按以下 JSON 格式返回,不要任何额外解释:
{"risk_level":"low","issues":[{"field":"","problem":"","suggestion":""}],"allow_video_generation":true}
risk_level 取值 low/medium/high。存在高风险违规时 allow_video_generation 设为 false。`;
export async function checkCompliance(
summary: ProductSummary,
selling: SellingPointResult,
storyboard: Storyboard,
signal?: AbortSignal,
): Promise<ComplianceCheck> {
const userContent = `【卖点】\n${JSON.stringify(selling, null, 2)}\n\n【分镜文案/旁白/字幕】\n${JSON.stringify(storyboard.scenes.map((s) => ({ subtitle: s.subtitle, voiceover: s.voiceover })), null, 2)}\n\n【风险点】\n${summary.risk_notes.join("、")}`;
const text = await chat(COMPLIANCE_PROMPT, userContent, { signal });
return extractJson(text) as ComplianceCheck;
}