import torch from diffsynth.pipelines.minimax_h3_audio_video import MiniMaxH3Pipeline, ModelConfig from diffsynth.utils.data.audio_video import write_video_audio from diffsynth.utils.data import VideoData from modelscope import dataset_snapshot_download vram_config = { "offload_dtype": torch.bfloat16, "offload_device": "cpu", "onload_dtype": torch.bfloat16, "onload_device": "cpu", "preparing_dtype": torch.bfloat16, "preparing_device": "cuda", "computation_dtype": torch.bfloat16, "computation_device": "cuda", } pipe = MiniMaxH3Pipeline.from_pretrained( torch_dtype=torch.bfloat16, device="cuda", model_configs=[ ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/text_encoder/model*.safetensors", **vram_config), ModelConfig(model_id="Comfy-Org/MiniMax-H3", origin_file_pattern="diffusion_models/minimax_h3_fl2va_pruned_bf16.safetensors", **vram_config), ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/video_vae/source/model.safetensors", **vram_config), ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/audio_vae/model.safetensors", **vram_config), ], vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 2, ) dataset_snapshot_download( dataset_id="DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", allow_file_pattern="minimax_h3/MiniMax-H3-Pruned-FL2VA/*", ) dataset_base_path = "data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Pruned-FL2VA" height, width, num_frames = 480, 832, 124 prompt = "A girl is very happy, she is speaking in english: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" pipe.load_lora(pipe.dit, "models/train/MiniMax-H3-T2VA-split/epoch-4.safetensors") video, audio = pipe( prompt=prompt, height=height, width=width, num_frames=num_frames, num_inference_steps=50, seed=0, ) write_video_audio( video=video, audio=audio, output_path="minimax_h3_t2va_lora.mp4", fps=24, audio_sample_rate=pipe.audio_vae.sample_rate, ) print("saved minimax_h3_t2va_lora.mp4", "frames:", len(video), "audio:", tuple(audio.shape)) frames = VideoData(f"{dataset_base_path}/video.mp4", height=height, width=width).raw_data() pipe.clear_lora() pipe.load_lora(pipe.dit, "models/train/MiniMax-H3-FL2VA-split/epoch-4.safetensors") video, audio = pipe( prompt=prompt, height=height, width=width, num_frames=num_frames, num_inference_steps=50, seed=0, keyframes=[frames[0], frames[num_frames - 1]], keyframe_indices=[0, -1], ) write_video_audio( video=video, audio=audio, output_path="minimax_h3_fl2va_lora.mp4", fps=24, audio_sample_rate=pipe.audio_vae.sample_rate, ) print("saved minimax_h3_fl2va_lora.mp4", "frames:", len(video), "audio:", tuple(audio.shape))