import torch from PIL import Image from diffsynth.pipelines.minimax_h3_audio_video import MiniMaxH3Pipeline, ModelConfig from diffsynth.utils.data.audio_video import write_video_audio, read_video_audio from diffsynth.utils.data.audio import read_audio from modelscope import dataset_snapshot_download def align_frame_count(frame_count): current = max(int(frame_count), 1) while current % 17 != 5: current += 1 return current vram_config = { "offload_dtype": torch.bfloat16, "offload_device": "cpu", "onload_dtype": torch.bfloat16, "onload_device": "cpu", "preparing_dtype": torch.bfloat16, "preparing_device": "cuda", "computation_dtype": torch.bfloat16, "computation_device": "cuda", } pipe = MiniMaxH3Pipeline.from_pretrained( torch_dtype=torch.bfloat16, device="cuda", model_configs=[ ModelConfig(model_id="Comfy-Org/MiniMax-H3", origin_file_pattern="diffusion_models/minimax_h3_ref2va_int8_convrot.safetensors", **vram_config), ModelConfig(model_id="Comfy-Org/MiniMax-H3", origin_file_pattern="text_encoders/qwen3vl_32b_minimax_h3_int8_convrot.safetensors", **vram_config), ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="Ref2VA/video_vae/source/model.safetensors", **vram_config), ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="Ref2VA/audio_vae/model.safetensors", **vram_config), ], processor_config=ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="Ref2VA/processor/"), vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 2, ) # Text + Reference Image -> Video + Audio dataset_snapshot_download(dataset_id="DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", allow_file_pattern="minimax_h3/MiniMax-H3-Ref2VA/*") ref_image = Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Ref2VA/0.png").convert("RGB") prompt = "一个网站页面,网站页面UI设计,网站动效,视频展示了流畅的网页向下滚动效果。一个极具爆发力与动感的产品官网风格产品落地页 UI/UX 演示视频,核心展示主体是该产品图片1。页面采用粗犷有力、倾斜的超大号无衬线字体进行张扬的排版。背景有极具速度感的动态光影、暗色碳纤维或运动透气网眼纹理在交织变换。视频展示了节奏紧凑、充满力量感的网页向下滚动效果,以及鼠标悬停时强烈的视觉放大与颜色反转等 UI 交互动作。" video, audio = pipe( prompt=prompt, height=480, width=832, num_frames=124, num_inference_steps=50, seed=42, references=[{"type": "image", "image": Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Ref2VA/0.png").convert("RGB")}] ) write_video_audio( video=video, audio=audio, output_path="ti2va.mp4", fps=24, audio_sample_rate=32000, ) # Text + Reference Audio + Reference Video -> Video + Audio ref_video, ref_video_audio, ref_video_sample_rate = read_video_audio( "data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Ref2VA/video.mp4", height=480, width=832, num_frames=124, fps=24, audio_sample_rate=pipe.audio_vae.sample_rate, ) ref_voice, ref_voice_sample_rate = read_audio( "data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Ref2VA/voice.mp3", duration=len(ref_video) / 24, resample=True, resample_rate=pipe.audio_vae.sample_rate, ) prompt = "subject_definitions:\n is the young man with short wavy blonde hair, wearing a bright pink suit jacket, matching pink trousers, an unbuttoned white shirt, and silver rings, holding a small black lamb in his arms in