import { ArrowLeftOutlined, AudioOutlined, CameraOutlined, CloseCircleOutlined, ColumnWidthOutlined, CustomerServiceOutlined, DeleteOutlined, DownloadOutlined, EditOutlined, FontSizeOutlined, InboxOutlined, PlayCircleOutlined, RightOutlined, ScissorOutlined, SwapOutlined, UserOutlined, } from "@ant-design/icons"; import { useCallback, useEffect, useRef, useState, type DragEvent } from "react"; import "../../styles/pages/more-tools.css"; import "../../styles/pages/image-workbench.css"; import { aiGenerationClient } from "../../api/aiGenerationClient"; import { uploadAssetWithProgress } from "../../api/uploadWithProgress"; import { waitForTask } from "../../api/taskSubscription"; import { saveToolTaskState, loadToolTaskState, clearToolTaskState } from "../workbench/toolKeepalive"; import { getServerBaseUrl } from "../../api/serverConnection"; import type { CreatePreviewTaskInput } from "../../api/webGenerationGateway"; import StudioToolLayout from "../../components/StudioToolLayout"; import type { WebGenerationPreviewTask, WebImageWorkbenchTool, WebViewKey } from "../../types"; import { addToolResultToAssetLibrary, saveToolResultToLocal } from "../workbench/toolResultActions"; interface DigitalHumanPageProps { isAuthenticated: boolean; isAdmin?: boolean; onCreateTask: (input: CreatePreviewTaskInput) => Promise; onRequireLogin: (input: { title: string; type: WebGenerationPreviewTask["type"]; prompt: string; }) => void; onOpenMore?: () => void; onOpenImageTool?: (tool: WebImageWorkbenchTool) => void; onSelectView?: (view: WebViewKey) => void; } function formatFileSize(bytes: number): string { if (!Number.isFinite(bytes) || bytes <= 0) return "0 KB"; if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(2)} MB`; return `${(bytes / 1024).toFixed(1)} KB`; } function summarizeUrl(value: string): string { try { const url = new URL(value); const lastSegment = url.pathname.split("/").filter(Boolean).pop(); return lastSegment ? `${url.host}/.../${lastSegment}` : url.host; } catch { return value.length > 72 ? `${value.slice(0, 34)}...${value.slice(-28)}` : value; } } function getGenerationAssetUrl(asset: { url: string; signedUrl?: string }): string { return asset.signedUrl || asset.url; } function getCurrentApiBaseLabel(): string { return getServerBaseUrl() || "/api"; } function DigitalHumanPage({ isAuthenticated, onCreateTask, onRequireLogin, onOpenMore, onOpenImageTool, onSelectView, }: DigitalHumanPageProps) { const [imageName, setImageName] = useState(""); const [imageFile, setImageFile] = useState(null); const [imagePreview, setImagePreview] = useState(""); const [audioName, setAudioName] = useState(""); const [audioFile, setAudioFile] = useState(null); const [audioPreview, setAudioPreview] = useState(""); const [promptInput, setPromptInput] = useState(""); const [watermark, setWatermark] = useState(false); const [keepOriginalAudio, setKeepOriginalAudio] = useState(true); const [notice, setNotice] = useState("等待上传参考图和音频"); const [isCreating, setIsCreating] = useState(false); const [isDownloadingResult, setIsDownloadingResult] = useState(false); const [isSavingResultAsset, setIsSavingResultAsset] = useState(false); const [resultVideoUrl, setResultVideoUrl] = useState(""); const [activeTaskId, setActiveTaskId] = useState(""); const [taskProgress, setTaskProgress] = useState(0); const pollRunRef = useRef(0); const cancelRef = useRef(false); const activeTaskIdRef = useRef(activeTaskId); activeTaskIdRef.current = activeTaskId; const keepaliveRestoredRef = useRef(false); const [isDragging, setIsDragging] = useState(false); const imageInputRef = useRef(null); const audioInputRef = useRef(null); const [isCanvasDragging, setIsCanvasDragging] = useState(false); useEffect(() => { return () => { if (imagePreview) URL.revokeObjectURL(imagePreview); }; }, [imagePreview]); useEffect(() => { return () => { if (audioPreview) URL.revokeObjectURL(audioPreview); }; }, [audioPreview]); // Keep-alive: restore saved task on mount useEffect(() => { if (keepaliveRestoredRef.current) return; keepaliveRestoredRef.current = true; const saved = loadToolTaskState("digital-human"); if (!saved || saved.resultUrl) return; setIsCreating(true); cancelRef.current = false; pollRunRef.current += 1; setActiveTaskId(saved.taskId); void waitForTaskResult(saved.taskId).catch(() => {}); setNotice("正在恢复数字人任务..."); }, []); useEffect(() => { return () => { pollRunRef.current += 1; cancelRef.current = true; }; }, []); const fileToDataUrl = (file: File) => new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(String(reader.result || "")); reader.onerror = () => reject(new Error("读取素材失败")); reader.readAsDataURL(file); }); const handleCancel = useCallback(() => { cancelRef.current = true; if (activeTaskId) { aiGenerationClient.cancelTask(activeTaskId).catch(() => {}); } setIsCreating(false); setNotice("已取消"); }, [activeTaskId]); const handleDragOver = (e: DragEvent) => { e.preventDefault(); if (e.dataTransfer?.types?.includes("Files")) setIsDragging(true); }; const handleDragLeave = (e: DragEvent) => { e.preventDefault(); if (!e.currentTarget.contains(e.relatedTarget as Node)) setIsDragging(false); }; const handleDrop = (e: DragEvent) => { e.preventDefault(); setIsDragging(false); const file = e.dataTransfer?.files?.[0]; if (!file) return; if (file.type.startsWith("image/")) { if (imagePreview) URL.revokeObjectURL(imagePreview); setImageName(file.name); setImageFile(file); setImagePreview(URL.createObjectURL(file)); setNotice(`已拖放参考图 ${file.name}`); } else if (file.type.startsWith("audio/")) { if (audioPreview) URL.revokeObjectURL(audioPreview); setAudioName(file.name); setAudioFile(file); setAudioPreview(URL.createObjectURL(file)); setNotice(`已拖放音频 ${file.name}`); } }; const handleCanvasDragOver = (e: DragEvent) => { e.preventDefault(); e.stopPropagation(); if (e.dataTransfer.types.includes("Files")) setIsCanvasDragging(true); }; const handleCanvasDragLeave = (e: DragEvent) => { e.preventDefault(); e.stopPropagation(); if (e.currentTarget === e.target || !e.currentTarget.contains(e.relatedTarget as Node)) setIsCanvasDragging(false); }; const handleCanvasDrop = (e: DragEvent) => { e.preventDefault(); e.stopPropagation(); setIsCanvasDragging(false); const file = e.dataTransfer.files[0]; if (!file) return; if (file.type.startsWith("image/")) { if (imagePreview) URL.revokeObjectURL(imagePreview); setImageName(file.name); setImageFile(file); setImagePreview(URL.createObjectURL(file)); pushDebugEntry("选择图片", `${file.name} / ${file.type || "unknown"} / ${formatFileSize(file.size)}`); setNotice(`已拖放参考图 ${file.name}`); } else if (file.type.startsWith("audio/")) { if (audioPreview) URL.revokeObjectURL(audioPreview); setAudioName(file.name); setAudioFile(file); setAudioPreview(URL.createObjectURL(file)); pushDebugEntry("选择音频", `${file.name} / ${file.type || "unknown"} / ${formatFileSize(file.size)}`); setNotice(`已拖放音频 ${file.name}`); } }; const handleCanvasClick = () => { if (!imagePreview) { imageInputRef.current?.click(); } else if (!audioPreview) { audioInputRef.current?.click(); } }; const handleDownloadResult = async () => { if (!resultVideoUrl || isDownloadingResult) return; setIsDownloadingResult(true); try { const status = await saveToolResultToLocal({ url: resultVideoUrl, name: `digital-human-${Date.now()}`, type: "video", isVideo: true, taskId: activeTaskId || undefined, }); setNotice(status === "saved" ? "已保存到本地" : "已开始保存到本地"); } catch (error) { setNotice(error instanceof Error ? error.message : "保存本地失败"); } finally { setIsDownloadingResult(false); } }; const handleAddResultToAssets = async () => { if (!resultVideoUrl || isSavingResultAsset) return; setIsSavingResultAsset(true); try { const status = await addToolResultToAssetLibrary({ url: resultVideoUrl, name: `数字人-${Date.now()}`, type: "video", isVideo: true, taskId: activeTaskId || undefined, description: "从工具盒数字人生成功能加入的视频。", tags: ["工具盒", "数字人", "生成视频"], }); setNotice(status === "server" ? "已加入资产库" : "已加入本地资产库"); } catch (error) { setNotice(error instanceof Error ? error.message : "加入资产库失败"); } finally { setIsSavingResultAsset(false); } }; const pushDebugEntry = ( label: string, detail: string, level: "info" | "success" | "error" = "info", data?: Record, ) => { console.debug("[DigitalHuman]", { stage: label, detail, level, ...(data ? { data } : {}) }); }; const waitForTaskResult = async (taskId: string): Promise => { const runId = ++pollRunRef.current; setActiveTaskId(taskId); setTaskProgress(0); saveToolTaskState("digital-human", { taskId, status: "running", progress: 0 }); pushDebugEntry("开始订阅", `开始接收任务 ${taskId} 的生成结果。`); const resultUrl = await waitForTask(taskId, { abortRef: cancelRef, onProgress: (e) => { if (pollRunRef.current !== runId) return; const progress = Math.max(0, Math.min(100, Math.trunc(e.progress || 0))); setTaskProgress(progress); setNotice(`任务 ${taskId} ${e.status},进度 ${progress}%`); pushDebugEntry( "状态更新", `${e.status} / ${progress}%${e.resultUrl ? ` / ${summarizeUrl(e.resultUrl)}` : ""}`, e.status === "failed" ? "error" : e.status === "completed" ? "success" : "info", { taskId, status: e }, ); if (e.status === "completed" && e.resultUrl) { setResultVideoUrl(e.resultUrl); setNotice(`任务完成,结果已接收:${taskId}`); clearToolTaskState("digital-human"); pushDebugEntry("结果已接收", summarizeUrl(e.resultUrl), "success", { taskId, resultUrl: e.resultUrl }); } }, }); if (cancelRef.current) throw new Error("已取消"); if (pollRunRef.current !== runId) throw new Error("当前轮询已被新的任务替换。"); if (!resultUrl) throw new Error("任务已完成,但服务端没有返回 resultUrl。"); return resultUrl; }; const handleCreateTask = async () => { if (isCreating) { pushDebugEntry("重复提交", "当前已有提交流程正在进行,已忽略本次点击。"); return; } const taskInput = { title: "数字人口播预览", type: "digital-human" as const, prompt: promptInput.trim() || `参考图: ${imageName || "未上传"}; 音频: ${audioName || "未上传"}; 生成数字人预览。`, }; if (!isAuthenticated) { onRequireLogin(taskInput); pushDebugEntry("登录检查", "当前没有有效登录态,已打开登录拦截。", "error"); setNotice("请先登录后再创建数字人预览任务。"); return; } if (!imageFile || !audioFile) { pushDebugEntry("素材检查", "缺少参考人像或音频源,无法提交。", "error", { hasImage: Boolean(imageFile), hasAudio: Boolean(audioFile), }); setNotice("请先上传参考人像和音频。"); return; } setIsCreating(true); setResultVideoUrl(""); setTaskProgress(0); cancelRef.current = false; pushDebugEntry("开始提交", `图片 ${imageFile.name},音频 ${audioFile.name}`, "info", { image: { name: imageFile.name, type: imageFile.type, size: imageFile.size }, audio: { name: audioFile.name, type: audioFile.type, size: audioFile.size }, }); setNotice("正在上传素材..."); try { pushDebugEntry( "读取素材", `${formatFileSize(imageFile.size)} 图片 / ${formatFileSize(audioFile.size)} 音频`, ); const [imageDataUrl, audioDataUrl] = await Promise.all([ fileToDataUrl(imageFile), fileToDataUrl(audioFile), ]); pushDebugEntry("读取完成", `DataURL 已生成:图片 ${formatFileSize(imageDataUrl.length)},音频 ${formatFileSize(audioDataUrl.length)}`); pushDebugEntry("上传素材", "正在上传参考图和音频到 OSS..."); const [imageAsset, audioAsset] = await Promise.all([ uploadAssetWithProgress( { dataUrl: imageDataUrl, name: imageFile.name, mimeType: imageFile.type || "image/png" }, { onProgress: (p) => setNotice(`上传参考图 ${p}%`) }, ), uploadAssetWithProgress( { dataUrl: audioDataUrl, name: audioFile.name, mimeType: audioFile.type || "audio/mpeg" }, { onProgress: (p) => setNotice(`上传音频 ${p}%`) }, ), ]); const missingSignedAssets = [ imageAsset.signedUrl ? "" : "图片", audioAsset.signedUrl ? "" : "音频", ].filter(Boolean); if (missingSignedAssets.length) { const apiBase = getCurrentApiBaseLabel(); const message = `服务端上传接口未返回 signedUrl,${missingSignedAssets.join("、")}素材不能直接交给 DashScope。请先更新并重启 key server:${apiBase}`; pushDebugEntry("服务端未更新", message, "error", { apiBase, imageUrl: imageAsset.url, audioUrl: audioAsset.url, hasImageSignedUrl: Boolean(imageAsset.signedUrl), hasAudioSignedUrl: Boolean(audioAsset.signedUrl), }); throw new Error(message); } setNotice("素材已上传,正在提交 wan2.2-s2v 任务..."); pushDebugEntry( "上传完成", `图片 ${summarizeUrl(getGenerationAssetUrl(imageAsset))};音频 ${summarizeUrl(getGenerationAssetUrl(audioAsset))}`, "success", { imageUrl: imageAsset.url, imageSignedUrl: imageAsset.signedUrl ? summarizeUrl(imageAsset.signedUrl) : "", audioUrl: audioAsset.url, audioSignedUrl: audioAsset.signedUrl ? summarizeUrl(audioAsset.signedUrl) : "", }, ); const imageGenerationUrl = getGenerationAssetUrl(imageAsset); const audioGenerationUrl = getGenerationAssetUrl(audioAsset); const videoTaskPayload = { model: "wan2.2-s2v", prompt: taskInput.prompt, imageUrl: imageGenerationUrl, audioUrl: audioGenerationUrl, quality: "720p", style: "speech", referenceUrls: [imageGenerationUrl], }; pushDebugEntry("提交任务", "正在请求 /api/ai/video,模型 wan2.2-s2v。", "info", { ...videoTaskPayload, imageUrl: summarizeUrl(imageGenerationUrl), audioUrl: summarizeUrl(audioGenerationUrl), referenceUrls: [summarizeUrl(imageGenerationUrl)], }); const { taskId } = await aiGenerationClient.createVideoTask(videoTaskPayload); pushDebugEntry("任务已创建", `DashScope 服务端任务 ID:${taskId}`, "success", { taskId }); await onCreateTask({ ...taskInput, params: { existingTaskId: taskId, model: "wan2.2-s2v", quality: "720p", referenceUrls: [imageAsset.url], }, title: "wan2.2-s2v 数字人口播", prompt: `${taskInput.prompt} 任务ID: ${taskId}`, }); setNotice(`任务已提交,服务端将使用 DashScope 并发池生成。任务ID:${taskId}`); pushDebugEntry("队列已同步", "页面任务队列已记录这次数字人任务。", "success", { taskId }); await waitForTaskResult(taskId); } catch (error) { const debugErrorMessage = error instanceof Error ? error.message : String(error || "任务创建失败"); pushDebugEntry("提交失败", debugErrorMessage, "error", { message: debugErrorMessage }); console.error("[DigitalHuman] submit failed", error); setNotice(error instanceof Error ? error.message : "任务创建失败,请稍后重试。"); } finally { setIsCreating(false); } }; return (
数字人
{imageName && audioName ? "素材已就绪" : "口播预览"} {imageName || audioName ? "人像 + 音频" : "待上传"}
{isDragging ? (
释放文件以上传
) : null}
参考人像 {imageName ? "已就绪" : "待上传"}
音频源 {audioName ? "已就绪" : "待上传"}
{audioPreview ?
} canvas={ resultVideoUrl ? (
) : imagePreview ? (
参考人像
) : (
{ if (e.key === "Enter" || e.key === " ") handleCanvasClick(); }} >
上传参考人像与音频
点击或拖拽上传;支持图片 (PNG/JPG/WEBP) 和音频 (MP3/WAV/M4A)
) } rightPanel={ <>
参数
提示词