import torch import json from diffsynth.utils.data import save_video, VideoData from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig from modelscope import dataset_snapshot_download from diffsynth.core.quant import QuantizeConfig vram_config = { "offload_dtype": torch.bfloat16, "offload_device": "cpu", "onload_dtype": torch.bfloat16, "onload_device": "cpu", "preparing_dtype": torch.bfloat16, "preparing_device": "cuda", "computation_dtype": torch.bfloat16, "computation_device": "cuda", "quantize": QuantizeConfig(method="entropack_lossy_quant_int8_raw"), } dataset_snapshot_download( dataset_id="DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", ) base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v" with open(f"{base}/t2v_example_1.json", "r", encoding="utf-8") as f: caption = json.load(f) pipe = LingBotVideoPipeline.from_pretrained( torch_dtype=torch.bfloat16, device="cuda", model_configs=[ ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="refiner/diffusion_pytorch_model*.safetensors", **vram_config), ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), ], processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 16, ) input_video = VideoData(f"{base}/video_1.mp4", height=1088, width=1920) video = pipe( prompt=caption, negative_prompt=pipe.default_negative_prompt, input_video=input_video, height=1088, width=1920, num_frames=81, num_inference_steps=8, cfg_scale=3.0, t_thresh=0.85, sigma_tail_steps=2, seed=0, ) save_video(video, "video_lingbot-video-moe-30b-a3b_t2v_refined.mp4", fps=15, quality=10)