''' !huggingface-cli download \ --repo-type dataset svjack/video-dataset-Lily-Bikini-rm-background-organized \ --local-dir video-dataset-Lily-Bikini-rm-background-organized import re def insert_content_in_string(insert_content, character_name, gender=None): """ 在原始字符串中特定位置插入内容。 :param insert_content: 要插入的内容 :param character_name: 角色名称 :param gender: 性别（可选，可以是 "1boy" 或 "1girl"） :return: 修改后的字符串 """ # 根据 character_name 和 gender 生成 original_string original_string = f"solo,{character_name}\(genshin impact\),{gender if gender else '1boy'},highres," # 根据 character_name 生成 target_pattern target_pattern = re.escape(character_name) # 插入内容 modified_string = re.sub(target_pattern, r'\g<0>' + insert_content, original_string) return original_string ,modified_string from datasets import load_dataset character_name = "Xiangling" gender = "1girl" # 可选参数 prompt_list = load_dataset("svjack/daily-actions-en-zh")["train"].to_pandas()["en"].map( lambda x: ", {}".format(x) ).map( lambda insert_content: insert_content_in_string(insert_content, character_name, gender)[-1] ).dropna().drop_duplicates().values.tolist() print(len(prompt_list)) import pandas as pd import pathlib reference_video_list = pd.Series( list(pathlib.Path("video-dataset-Lily-Bikini-rm-background-organized").rglob("*.mp4")) ).map(str).values.tolist() print(len(reference_video_list)) from itertools import product pd.DataFrame(list(product(*[reference_video_list, prompt_list])))[[1, 0]].rename( columns = { 1: "prompt", 0: "input_video" } ).to_csv("xiangling_video_seed.csv", index = False) !python produce_gif_script.py xiangling_video_seed.csv "svjack/GenshinImpact_XL_Base" xiangling_gif_dir \ --num_frames 16 --temp_folder temp_frames --seed 0 --controlnet_conditioning_scale 0.3 ''' import sys sys.path.insert(0, "diffusers-sdxl-controlnet/examples/community/") from animatediff_controlnet_sdxl import * import argparse from moviepy.editor import VideoFileClip, ImageSequenceClip import os import torch from diffusers.models import MotionAdapter from diffusers import DDIMScheduler, AutoPipelineForText2Image, ControlNetModel from diffusers.utils import export_to_gif from PIL import Image from controlnet_aux.processor import Processor import pandas as pd import random from tqdm import tqdm # 初始化 MotionAdapter 和 ControlNetModel adapter = MotionAdapter.from_pretrained("a-r-r-o-w/animatediff-motion-adapter-sdxl-beta", torch_dtype=torch.float16) def initialize_pipeline(model_id): scheduler = DDIMScheduler.from_pretrained(model_id, subfolder="scheduler", clip_sample=False, timestep_spacing="linspace", beta_schedule="linear", steps_offset=1) controlnet = ControlNetModel.from_pretrained("thibaud/controlnet-openpose-sdxl-1.0", torch_dtype=torch.float16).to("cuda") # 初始化 AnimateDiffSDXLControlnetPipeline pipe = AnimateDiffSDXLControlnetPipeline.from_pretrained( model_id, controlnet=controlnet, motion_adapter=adapter, scheduler=scheduler, torch_dtype=torch.float16, ).to("cuda") pipe.enable_vae_slicing() pipe.enable_vae_tiling() return pipe # 全局初始化管道 pipe = None def split_video_into_frames(input_video_path, num_frames, temp_folder='temp_frames'): """ 将视频处理成指定帧数的视频，并保持原始的帧率。 :param input_video_path: 输入视频文件路径 :param num_frames: 目标帧数 :param temp_folder: 临时文件夹路径 """ clip = VideoFileClip(input_video_path) original_duration = clip.duration segment_duration = original_duration / num_frames if not os.path.exists(temp_folder): os.makedirs(temp_folder) for i in range(num_frames): frame_time = i * segment_duration frame_path = os.path.join(temp_folder, f'frame_{i:04d}.png') clip.save_frame(frame_path, t=frame_time) frame_paths = [os.path.join(temp_folder, f'frame_{i:04d}.png') for i in range(num_frames)] final_clip = ImageSequenceClip(frame_paths, fps=clip.fps) final_clip.write_videofile("resampled_video.mp4", codec='libx264') print(f"新的视频已保存到 resampled_video.mp4，包含 {num_frames} 个帧，并保持原始的帧率。") def generate_video_with_prompt(input_video_path, prompt, model_id, gif_output_path, seed=0, num_frames=16, keep_imgs=False, temp_folder='temp_frames', num_inference_steps=50, guidance_scale=20, controlnet_conditioning_scale=0.5, width=512, height=768): """ 生成带有文本提示的视频。 :param input_video_path: 输入视频文件路径 :param prompt: 文本提示 :param model_id: 模型ID :param gif_output_path: GIF 输出文件路径 :param seed: 随机种子 :param num_frames: 目标帧数 :param keep_imgs: 是否保留临时图片 :param temp_folder: 临时文件夹路径 :param num_inference_steps: 推理步数 :param guidance_scale: 引导比例 :param controlnet_conditioning_scale: ControlNet 条件比例 :param width: 输出宽度 :param height: 输出高度 """ split_video_into_frames(input_video_path, num_frames, temp_folder) folder_path = temp_folder frames = os.listdir(folder_path) frames = list(filter(lambda x: x.endswith(".png"), frames)) frames.sort() conditioning_frames = list(map(lambda x: Image.open(os.path.join(folder_path, x)).resize((1024, 1024)), frames))[:num_frames] p2 = Processor("openpose") cn2 = [p2(frame) for frame in conditioning_frames] negative_prompt = "bad quality, worst quality, jpeg artifacts, ugly" generator = torch.Generator(device="cuda").manual_seed(seed) global pipe if pipe is None: pipe = initialize_pipeline(model_id) output = pipe( prompt=prompt, negative_prompt=negative_prompt, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, controlnet_conditioning_scale=controlnet_conditioning_scale, width=width, height=height, num_frames=num_frames, conditioning_frames=cn2, generator=generator ) frames = output.frames[0] export_to_gif(frames, gif_output_path) print(f"生成的 GIF 已保存到 {gif_output_path}") if not keep_imgs: # 删除临时文件夹 import shutil shutil.rmtree(temp_folder) def sanitize_prompt(prompt): """ 将提示词中的空格和非英文字符替换为下划线。 """ return "".join([c if c.isalnum() or c in [",", "，"] else '_' for c in prompt]) if __name__ == "__main__": parser = argparse.ArgumentParser(description="生成带有文本提示的视频") parser.add_argument("csv_file", help="CSV 文件路径") parser.add_argument("model_id", help="模型ID") parser.add_argument("output_dir", help="GIF 输出目录") parser.add_argument("--seed", type=int, default=0, help="随机种子") parser.add_argument("--num_frames", type=int, default=16, help="目标帧数") parser.add_argument("--keep_imgs", action="store_true", help="是否保留临时图片") parser.add_argument("--temp_folder", default='temp_frames', help="临时文件夹路径") parser.add_argument("--num_inference_steps", type=int, default=50, help="推理步数") parser.add_argument("--guidance_scale", type=float, default=20.0, help="引导比例") parser.add_argument("--controlnet_conditioning_scale", type=float, default=0.5, help="ControlNet 条件比例") parser.add_argument("--width", type=int, default=512, help="输出宽度") parser.add_argument("--height", type=int, default=768, help="输出高度") args = parser.parse_args() # 读取CSV文件 df = pd.read_csv(args.csv_file) for index, row in tqdm(df.iterrows(), total=df.shape[0]): input_video = row['input_video'] prompt = row['prompt'] # 随机设定seed seed = random.randint(0, 2**32 - 1) # 处理提示词 sanitized_prompt = sanitize_prompt(prompt) # 生成GIF输出路径，包含seed if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) gif_output_path = os.path.join(args.output_dir, f"{sanitized_prompt}_seed_{seed}.gif") generate_video_with_prompt(input_video, prompt, args.model_id, gif_output_path, seed, args.num_frames, args.keep_imgs, args.temp_folder, args.num_inference_steps, args.guidance_scale, args.controlnet_conditioning_scale, args.width, args.height)