import torch from diffsynth.pipelines.minimax_h3_audio_video import MiniMaxH3Pipeline, ModelConfig from diffsynth.utils.data.audio_video import write_video_audio from diffsynth.utils.data import VideoData from modelscope import dataset_snapshot_download vram_config = { "offload_dtype": torch.bfloat16, "offload_device": "cpu", "onload_dtype": torch.bfloat16, "onload_device": "cpu", "preparing_dtype": torch.bfloat16, "preparing_device": "cuda", "computation_dtype": torch.bfloat16, "computation_device": "cuda", } dataset_snapshot_download( dataset_id="DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", allow_file_pattern="minimax_h3/MiniMax-H3-FL2VA/*", ) dataset_base_path = "data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-FL2VA" height, width, num_frames = 480, 832, 124 prompt = "A girl is very happy, she is speaking in english: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" frames = VideoData(f"{dataset_base_path}/video.mp4", height=height, width=width).raw_data() def build_pipe(dit_path): return MiniMaxH3Pipeline.from_pretrained( torch_dtype=torch.bfloat16, device="cuda", model_configs=[ ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/text_encoder/model*.safetensors", **vram_config), ModelConfig(path=dit_path, **vram_config), ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/video_vae/source/model.safetensors", **vram_config), ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/audio_vae/model.safetensors", **vram_config), ], vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 2, ) pipe = build_pipe("models/train/MiniMax-H3-T2VA-full/epoch-1.safetensors") video, audio = pipe( prompt=prompt, height=height, width=width, num_frames=num_frames, num_inference_steps=50, seed=0, ) write_video_audio( video=video, audio=audio, output_path="minimax_h3_t2va_full.mp4", fps=24, audio_sample_rate=pipe.audio_vae.sample_rate, ) print("saved minimax_h3_t2va_full.mp4", "frames:", len(video), "audio:", tuple(audio.shape)) del pipe torch.cuda.empty_cache() pipe = build_pipe("models/train/MiniMax-H3-FL2VA-full/epoch-1.safetensors") video, audio = pipe( prompt=prompt, height=height, width=width, num_frames=num_frames, num_inference_steps=50, seed=0, keyframes=[frames[0], frames[num_frames - 1]], keyframe_indices=[0, -1], ) write_video_audio( video=video, audio=audio, output_path="minimax_h3_fl2va_full.mp4", fps=24, audio_sample_rate=pipe.audio_vae.sample_rate, ) print("saved minimax_h3_fl2va_full.mp4", "frames:", len(video), "audio:", tuple(audio.shape))