Storyboard to animatic

"""A one-line idea to a narrated animatic: storyboard, one image per shot in a shared style, motion, voice, edit.

pip install openai requests      (and ffmpeg on PATH)
RUNCRATE_API_KEY=... VIDEO_SHOTS=1 python storyboard_to_animatic.py "A lighthouse keeper finds a robot washed up on the shore"
VIDEO_SHOTS is how many shots get generated motion; the rest get a slow push-in on the still (free, and the norm for an animatic).
"""
import base64, json, os, subprocess, sys, time
from pathlib import Path

import requests
from openai import OpenAI

API = "https://api.runcrate.ai/v1"
KEY = os.environ["RUNCRATE_API_KEY"]
client = OpenAI(base_url=API, api_key=KEY, max_retries=4, timeout=180)
WRITER = "openai/gpt-oss-120b"                 # on the free tier
FRAMES = "black-forest-labs/FLUX-2-klein-4b"   # takes reference images: every shot after the first sees shot 1
MOTION = "bytedance/seedance-2.0-fast"
TTS, VOICE = "hexgrad/Kokoro-82M", "bm_george"
VIDEO_SHOTS = int(os.environ.get("VIDEO_SHOTS", "1"))
OUT = Path("out"); OUT.mkdir(exist_ok=True)
ledger = []

def log(step, model, t0, cost):
    ledger.append({"step": step, "model": model, "seconds": round(time.time() - t0, 2), "cost_usd": cost})
    print(f"{step:<10} {model:<36} {time.time() - t0:7.2f}s  ${cost:.6f}")

def ffmpeg(*args):
    subprocess.run(["ffmpeg", "-y", "-loglevel", "error", *args], check=True)

def duration(path: Path) -> float:
    return float(subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(path)],
                                capture_output=True, text=True).stdout)

def data_url(path: Path) -> str:
    b = path.read_bytes()
    mime = "image/png" if b[:4] == b"\x89PNG" else "image/webp" if b[8:12] == b"WEBP" else "image/jpeg"
    return f"data:{mime};base64,{base64.b64encode(b).decode()}"

def video(body: dict, path: Path) -> float:
    """Submit, poll with backoff (riding out 429s and 5xx), download. Returns the billed cost."""
    h = {"Authorization": f"Bearer {KEY}"}
    for attempt in range(5):
        res = requests.post(f"{API}/videos", json=body, headers=h, timeout=60)
        if res.status_code not in (429, 500, 502, 503, 504):
            break
        time.sleep(int(res.headers.get("Retry-After", 2 ** attempt)))
    res.raise_for_status()
    job, cost, wait = res.json(), float(res.headers.get("x-llm-cost", 0)), 5
    while job["status"] not in ("completed", "failed"):
        time.sleep(wait)
        r = requests.get(f"{API}/videos/{job['id']}", headers=h, timeout=30)
        if r.status_code in (429, 500, 502, 503, 504):
            wait = min(wait * 2, 30); continue
        r.raise_for_status()
        job, wait = r.json(), min(wait * 1.5, 20)
    if job["status"] == "failed":
        raise RuntimeError(f"video {job['id']} failed: {job.get('error')}")
    path.write_bytes(requests.get(f"{API}/videos/{job['id']}/download", headers=h, timeout=300).content)
    return cost

def main(idea: str) -> None:
    t0 = time.time()
    raw = client.chat.completions.with_raw_response.create(
        model=WRITER, temperature=0.7,
        messages=[{"role": "user", "content": f"Storyboard a 3-shot animatic for: {idea}. One visual style for every shot. "
                   "Each shot: an image prompt (who, where, framing; no text in frame), a camera-motion prompt, and narration of at most 14 words."}],
        response_format={"type": "json_schema", "json_schema": {"name": "board", "strict": True, "schema": {
            "type": "object", "additionalProperties": False, "required": ["title", "style", "shots"],
            "properties": {"title": {"type": "string"}, "style": {"type": "string"},
                           "shots": {"type": "array", "items": {"type": "object", "additionalProperties": False,
                                     "required": ["image_prompt", "motion_prompt", "narration"],
                                     "properties": {"image_prompt": {"type": "string"}, "motion_prompt": {"type": "string"}, "narration": {"type": "string"}}}}}}}},
    )
    board = json.loads(raw.parse().choices[0].message.content)
    log("board", WRITER, t0, float(raw.headers.get("x-llm-cost", 0)))
    (OUT / "board.json").write_text(json.dumps(board, indent=2))

    clips = []
    for n, shot in enumerate(board["shots"][:3], 1):
        still = OUT / f"shot-{n}.jpg"
        t0 = time.time()
        body = {"aspect_ratio": "16:9"}
        if n > 1:  # keep characters and palette from shot 1
            body["images"] = [data_url(OUT / "shot-1.jpg")]
        raw = client.images.with_raw_response.generate(model=FRAMES, prompt=f"{board['style']}. {shot['image_prompt']}", extra_body=body)
        url = raw.parse().data[0].url
        still.write_bytes(base64.b64decode(url.split(",", 1)[1]) if url.startswith("data:") else requests.get(url, timeout=60).content)
        log(f"frame {n}", FRAMES, t0, float(raw.headers.get("x-llm-cost", 0)))

        t0 = time.time()
        raw = client.audio.speech.with_raw_response.create(model=TTS, voice=VOICE, input=shot["narration"], response_format="wav")
        voice = OUT / f"voice-{n}.wav"; voice.write_bytes(raw.content)
        log(f"voice {n}", TTS, t0, float(raw.headers.get("x-llm-cost", 0)))
        length = max(4.0, duration(voice) + 0.6)

        t0 = time.time()
        motion = OUT / f"motion-{n}.mp4"
        if n <= VIDEO_SHOTS:
            cost = video({"model": MOTION, "prompt": shot["motion_prompt"], "duration": 4, "resolution": "480p",
                          "generate_audio": False, "image": data_url(still)}, motion)
            log(f"motion {n}", MOTION, t0, cost)
            src = ["-stream_loop", "-1", "-i", str(motion)]
        else:
            ffmpeg("-loop", "1", "-i", str(still), "-vf",
                   f"scale=2560:-2,zoompan=z='min(zoom+0.0008,1.15)':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d={int(length * 30)}:s=1280x720:fps=30",
                   "-t", f"{length}", "-pix_fmt", "yuv420p", str(motion))
            log(f"push-in {n}", "ffmpeg zoompan", t0, 0.0)
            src = ["-i", str(motion)]

        clip = OUT / f"clip-{n}.mp4"
        ffmpeg(*src, "-i", str(voice), "-filter_complex",
               "[0:v]scale=1280:720:force_original_aspect_ratio=increase,crop=1280:720,fps=30,setsar=1[v];[1:a]apad[a]",
               "-map", "[v]", "-map", "[a]", "-t", f"{length}", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-ar", "44100", str(clip))
        clips.append(clip)

    t0 = time.time()
    (OUT / "clips.txt").write_text("".join(f"file '{c.resolve()}'\n" for c in clips))
    ffmpeg("-f", "concat", "-safe", "0", "-i", str(OUT / "clips.txt"), "-c", "copy", str(OUT / "animatic.mp4"))
    log("edit", "ffmpeg concat", t0, 0.0)

    total = sum(s["cost_usd"] for s in ledger)
    Path("run.json").write_text(json.dumps({"title": board["title"], "steps": ledger, "total_usd": total, "seconds": duration(OUT / "animatic.mp4")}, indent=2))
    print(f"\nout/animatic.mp4  {duration(OUT / 'animatic.mp4'):.1f}s  total ${total:.4f}  {board['title']}")

if __name__ == "__main__":
    main(sys.argv[1])

// A one-line idea to a narrated animatic: storyboard, one image per shot in a shared style, motion, voice, edit.
//
// npm install openai      (and ffmpeg on PATH)
// RUNCRATE_API_KEY=... VIDEO_SHOTS=1 npx tsx storyboard-to-animatic.ts "A lighthouse keeper finds a robot washed up on the shore"
// VIDEO_SHOTS is how many shots get generated motion; the rest get a slow push-in on the still (free, and the norm for an animatic).
import { execFileSync } from "node:child_process";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import OpenAI from "openai";

const API = "https://api.runcrate.ai/v1";
const KEY = process.env.RUNCRATE_API_KEY ?? "";
const client = new OpenAI({ baseURL: API, apiKey: KEY, maxRetries: 4, timeout: 180_000 });
const WRITER = "openai/gpt-oss-120b"; // on the free tier
const FRAMES = "black-forest-labs/FLUX-2-klein-4b"; // takes reference images: every shot after the first sees shot 1
const MOTION = "bytedance/seedance-2.0-fast";
const TTS = "hexgrad/Kokoro-82M";
const VOICE = "bm_george";
const VIDEO_SHOTS = Number(process.env.VIDEO_SHOTS ?? "1");
const ledger: { step: string; model: string; seconds: number; cost_usd: number }[] = [];
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
const ffmpeg = (...args: string[]) => execFileSync("ffmpeg", ["-y", "-loglevel", "error", ...args]);
const duration = (path: string) => Number(execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path]).toString());
mkdirSync("out", { recursive: true });

function log(step: string, model: string, t0: number, cost: number) {
  const seconds = (Date.now() - t0) / 1000;
  ledger.push({ step, model, seconds, cost_usd: cost });
  console.log(`${step.padEnd(10)} ${model.padEnd(36)} ${seconds.toFixed(2).padStart(7)}s  $${cost.toFixed(6)}`);
}

function dataUrl(path: string): string {
  const b = readFileSync(path);
  const mime = b.subarray(0, 4).toString("latin1") === "\x89PNG" ? "image/png" : b.subarray(8, 12).toString() === "WEBP" ? "image/webp" : "image/jpeg";
  return `data:${mime};base64,${b.toString("base64")}`;
}

/** Submit, poll with backoff (riding out 429s and 5xx), download. Returns the billed cost. */
async function video(body: object, path: string): Promise<number> {
  const h = { Authorization: `Bearer ${KEY}`, "Content-Type": "application/json" };
  let res = await fetch(`${API}/videos`, { method: "POST", headers: h, body: JSON.stringify(body) });
  for (let attempt = 0; [429, 500, 502, 503, 504].includes(res.status) && attempt < 5; attempt++) {
    await sleep(Number(res.headers.get("retry-after") ?? 2 ** attempt) * 1000);
    res = await fetch(`${API}/videos`, { method: "POST", headers: h, body: JSON.stringify(body) });
  }
  if (!res.ok) throw new Error(`video submit answered ${res.status}: ${await res.text()}`);
  const cost = Number(res.headers.get("x-llm-cost") ?? 0);
  let job = (await res.json()) as { id: string; status: string; error?: unknown };
  let wait = 5000;
  while (job.status !== "completed" && job.status !== "failed") {
    await sleep(wait);
    const r = await fetch(`${API}/videos/${job.id}`, { headers: h });
    if ([429, 500, 502, 503, 504].includes(r.status)) {
      wait = Math.min(wait * 2, 30_000);
      continue;
    }
    if (!r.ok) throw new Error(`video poll answered ${r.status}`);
    job = (await r.json()) as typeof job;
    wait = Math.min(wait * 1.5, 20_000);
  }
  if (job.status === "failed") throw new Error(`video ${job.id} failed: ${JSON.stringify(job.error)}`);
  writeFileSync(path, Buffer.from(await (await fetch(`${API}/videos/${job.id}/download`, { headers: h })).arrayBuffer()));
  return cost;
}

type Board = { title: string; style: string; shots: { image_prompt: string; motion_prompt: string; narration: string }[] };
const idea = process.argv[2];

let t0 = Date.now();
const res = await client.chat.completions
  .create({
    model: WRITER,
    temperature: 0.7,
    messages: [
      {
        role: "user",
        content:
          `Storyboard a 3-shot animatic for: ${idea}. One visual style for every shot. ` +
          "Each shot: an image prompt (who, where, framing; no text in frame), a camera-motion prompt, and narration of at most 14 words.",
      },
    ],
    response_format: {
      type: "json_schema",
      json_schema: {
        name: "board",
        strict: true,
        schema: {
          type: "object",
          additionalProperties: false,
          required: ["title", "style", "shots"],
          properties: {
            title: { type: "string" },
            style: { type: "string" },
            shots: {
              type: "array",
              items: {
                type: "object",
                additionalProperties: false,
                required: ["image_prompt", "motion_prompt", "narration"],
                properties: { image_prompt: { type: "string" }, motion_prompt: { type: "string" }, narration: { type: "string" } },
              },
            },
          },
        },
      },
    },
  })
  .withResponse();
const board: Board = JSON.parse(res.data.choices[0].message.content ?? "{}");
log("board", WRITER, t0, Number(res.response.headers.get("x-llm-cost") ?? 0));
writeFileSync("out/board.json", JSON.stringify(board, null, 2));

const clips: string[] = [];
for (const [k, shot] of board.shots.slice(0, 3).entries()) {
  const n = k + 1;
  const still = `out/shot-${n}.jpg`;
  t0 = Date.now();
  const body: Record<string, unknown> = { model: FRAMES, prompt: `${board.style}. ${shot.image_prompt}`, aspect_ratio: "16:9" };
  if (n > 1) body.images = [dataUrl("out/shot-1.jpg")]; // keep characters and palette from shot 1
  const img = await client.images.generate(body as unknown as OpenAI.Images.ImageGenerateParamsNonStreaming).withResponse();
  const url = img.data.data?.[0]?.url ?? "";
  writeFileSync(still, url.startsWith("data:") ? Buffer.from(url.split(",", 2)[1], "base64") : Buffer.from(await (await fetch(url)).arrayBuffer()));
  log(`frame ${n}`, FRAMES, t0, Number(img.response.headers.get("x-llm-cost") ?? 0));

  t0 = Date.now();
  const speech = await client.audio.speech.create({ model: TTS, voice: VOICE, input: shot.narration, response_format: "wav" }).withResponse();
  const voice = `out/voice-${n}.wav`;
  writeFileSync(voice, Buffer.from(await speech.data.arrayBuffer()));
  log(`voice ${n}`, TTS, t0, Number(speech.response.headers.get("x-llm-cost") ?? 0));
  const length = Math.max(4, duration(voice) + 0.6);

  t0 = Date.now();
  const motion = `out/motion-${n}.mp4`;
  let src: string[];
  if (n <= VIDEO_SHOTS) {
    const cost = await video({ model: MOTION, prompt: shot.motion_prompt, duration: 4, resolution: "480p", generate_audio: false, image: dataUrl(still) }, motion);
    log(`motion ${n}`, MOTION, t0, cost);
    src = ["-stream_loop", "-1", "-i", motion];
  } else {
    ffmpeg(
      "-loop", "1", "-i", still, "-vf",
      `scale=2560:-2,zoompan=z='min(zoom+0.0008,1.15)':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=${Math.trunc(length * 30)}:s=1280x720:fps=30`,
      "-t", `${length}`, "-pix_fmt", "yuv420p", motion,
    );
    log(`push-in ${n}`, "ffmpeg zoompan", t0, 0);
    src = ["-i", motion];
  }
  const clip = `out/clip-${n}.mp4`;
  ffmpeg(
    ...src, "-i", voice, "-filter_complex",
    "[0:v]scale=1280:720:force_original_aspect_ratio=increase,crop=1280:720,fps=30,setsar=1[v];[1:a]apad[a]",
    "-map", "[v]", "-map", "[a]", "-t", `${length}`, "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-ar", "44100", clip,
  );
  clips.push(clip);
}

t0 = Date.now();
writeFileSync("out/clips.txt", clips.map((c) => `file '${resolve(c)}'\n`).join(""));
ffmpeg("-f", "concat", "-safe", "0", "-i", "out/clips.txt", "-c", "copy", "out/animatic.mp4");
log("edit", "ffmpeg concat", t0, 0);

const total = ledger.reduce((s, r) => s + r.cost_usd, 0);
writeFileSync("run.json", JSON.stringify({ title: board.title, steps: ledger, total_usd: total, seconds: duration("out/animatic.mp4") }, null, 2));
console.log(`\nout/animatic.mp4  ${duration("out/animatic.mp4").toFixed(1)}s  total $${total.toFixed(4)}  ${board.title}`);