基于多模态模型的视频 Prompt 生成实践

A cinematic night street scene featuring a lone person walking through rain-soaked pavement, neon reflections, blue and magenta lighting, handheld camera movement, moody atmosphere.

视频上传
  ↓
格式校验
  ↓
视频转码
  ↓
关键帧抽取
  ↓
多模态模型分析
  ↓
结构化结果整理
  ↓
Prompt 模板拼装
  ↓
结果返回

async function validateVideo(file) {
  const meta = await getVideoMeta(file);

  if (meta.duration > 120) {
    throw new Error('video duration exceeds limit');
  }

  if (file.size > 100 * 1024 * 1024) {
    throw new Error('video file is too large');
  }

  return meta;
}

async function extractKeyframes(videoPath) {
  const intervalFrames = await extractByInterval(videoPath, 2);
  const sceneFrames = await extractBySceneChange(videoPath, {
    threshold: 0.35
  });

  return removeDuplicateFrames([
    ...intervalFrames,
    ...sceneFrames
  ]);
}

{
  "frames": [
    {
      "time": "00:00",
      "summary": "A person stands in a neon-lit street at night."
    },
    {
      "time": "00:02",
      "summary": "The camera moves closer, rain reflections become clearer."
    }
  ]
}

{
  "subject": "a person walking through a rainy city street",
  "scene": "neon-lit street at night",
  "camera": "medium shot, subtle handheld movement",
  "lighting": "blue and magenta neon reflections",
  "motion": "slow walking motion",
  "mood": "moody, cinematic",
  "style": "realistic cyberpunk film look"
}

function formatImagePrompt(data) {
  return [
    data.subject,
    data.scene,
    data.lighting,
    data.mood,
    data.style,
    'cinematic composition'
  ].join(', ');
}

function formatVideoPrompt(data) {
  return [
    data.subject,
    data.motion,
    data.camera,
    data.scene,
    data.lighting,
    data.mood,
    data.style
  ].join(', ');
}

pending
processing
extracting
analyzing
formatting
completed
failed

{
  "taskId": "task_xxx",
  "status": "analyzing",
  "progress": 70
}

function getMaxFrameCount(duration) {
  if (duration <= 10) return 6;
  if (duration <= 30) return 12;
  return 20;
}

const hash = await getFileHash(file);
const cached = await findResultByHash(hash);

if (cached) {
  return cached;
}