零基础AI漫剧：用代码搭建从剧本到成片的自动化生产管线

import requests
import json

def generate_script(story_idea: str) -> list:
    prompt = f"""将以下故事转化为5个分镜，每个分镜包含：
    - scene_description: 画面描述（包含角色、背景、动作）
    - dialogue: 角色台词（若有）
    - duration_sec: 预估时长（3-8秒）
    输出JSON数组。
    故事：{story_idea}"""
    
    response = requests.post(
        "https://api.deepseek.com/v1/chat/completions",
        headers={"Authorization": "Bearer YOUR_API_KEY"},
        json={
            "model": "deepseek-chat",
            "messages": [{"role": "user", "content": prompt}],
            "response_format": {"type": "json_object"}
        }
    )
    return json.loads(response.json()["choices"][0]["message"]["content"])["scenes"]

# 示例
scenes = generate_script("一个机器人男孩在废弃城市中寻找花朵，最终发现一朵发光的花")
print(json.dumps(scenes, indent=2, ensure_ascii=False))

[
  {"id":1,"desc":"机器人男孩站立在废墟中，背景是灰暗的天空，他低头看着地面","dialogue":"这里什么都没有...","duration":5},
  {"id":2,"desc":"男孩抬头，远处有微光，他迈出一步","dialogue":"那是什么？","duration":4}
]

from diffusers import StableDiffusionXLPipeline, IPAdapter
import torch

pipe = StableDiffusionXLPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0",
    torch_dtype=torch.float16
).to("cuda")

# 加载IP-Adapter（支持FaceID风格）
pipe.load_ip_adapter(
    "h94/IP-Adapter",
    subfolder="sdxl_models",
    weight_name="ip-adapter-plus_sdxl_vit-h.safetensors"
)
pipe.set_ip_adapter_scale(0.8)  # 控制参考强度

# 加载参考图（角色正面照）
from PIL import Image
ref_image = Image.open("character_front.png").resize((512, 512))

# 生成每个分镜图
def generate_scene_image(prompt: str, ref_img: Image) -> Image:
    images = pipe(
        prompt=prompt,
        ip_adapter_image=ref_img,
        negative_prompt="low quality, blurry, distorted",
        num_inference_steps=30,
        width=832, height=480
    ).images
    return images[0]

# 批量生成
scene_images = []
for s in scenes:
    img = generate_scene_image(s["desc"], ref_image)
    scene_images.append(img)
    img.save(f"scene_{s['id']}.png")

from diffusers import AnimateDiffPipeline, MotionAdapter
from diffusers.utils import export_to_video

adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-2")
pipe_anim = AnimateDiffPipeline.from_pretrained(
    "SG161222/Realistic_Vision_V5.1_noVAE",
    motion_adapter=adapter,
    torch_dtype=torch.float16
).to("cuda")
# 启用FreeInit提升运动平滑度
pipe_anim.enable_free_init(num_steps=5)

def animate_scene(image: Image, prompt_plus: str, n_frames=24) -> List[Image]:
    # 将静态图作为初始噪声或控制条件（此处简化：直接生成随机运动）
    output = pipe_anim(
        prompt=f"{prompt_plus}, slow motion, gentle breeze",
        negative_prompt="static, no movement",
        num_frames=n_frames,
        guidance_scale=7.5,
        num_inference_steps=25
    ).frames[0]  # 返回列表[PIL]
    return output

# 仅对第1个场景做动画测试（实际可循环）
anim_frames = animate_scene(scene_images[0], scenes[0]["desc"])
export_to_video(anim_frames, "scene_1_anim.mp4", fps=8)

import edge_tts
import asyncio
import subprocess

async def text_to_speech(text: str, output_file: str):
    communicate = edge_tts.Communicate(text, "zh-CN-XiaoyiNeural")
    await communicate.save(output_file)

# 为每个分镜生成音频
audio_files = []
for s in scenes:
    if s.get("dialogue"):
        audio_path = f"audio_{s['id']}.mp3"
        asyncio.run(text_to_speech(s["dialogue"], audio_path))
        audio_files.append(audio_path)
    else:
        audio_files.append(None)

# 假设我们将所有动画片段拼接为单一视频（或分别合成）
# 此处展示单个分镜合成命令
def combine_av(video_path: str, audio_path: str, output_path: str):
    cmd = [
        "ffmpeg", "-i", video_path, "-i", audio_path,
        "-c:v", "libx264", "-c:a", "aac", "-shortest",
        "-pix_fmt", "yuv420p", output_path
    ]
    subprocess.run(cmd, check=True)

combine_av("scene_1_anim.mp4", "audio_1.mp3", "final_scene_1.mp4")