Build it in your project
Paste this line into your agent (Claude Code, Codex, Cursor, OpenCode, Cline). It sets up your key, then builds the pipeline in your project:Add Runcrate's Kling 2.6 to this project: run curl -fsSL "https://runcrate.ai/agents.md?model=kuaishou%2Fkling-v2.6&intent=build" and follow it. Then build: a script that turns one product photo into a narrated 7-second 1280x720 ad: Qwen3-VL facts, a gpt-oss-120b script, two FLUX.2 klein scene edits that keep the product, Kling v2.6 image-to-video, Kokoro narration, ffmpeg mux.
The code
One file per language. Both readRUNCRATE_API_KEY from the environment; nothing secret is in the code.
"""One product photo to a narrated 7-second video ad: vision, script, scene edits, image-to-video, voice, mux.
pip install openai requests (and ffmpeg on PATH)
RUNCRATE_API_KEY=... python product_photo_to_video_ad.py product.jpg
"""
import base64, json, os, subprocess, sys, time
from pathlib import Path
import requests
from openai import OpenAI
API = "https://api.runcrate.ai/v1"
KEY = os.environ["RUNCRATE_API_KEY"]
client = OpenAI(base_url=API, api_key=KEY, max_retries=4, timeout=180)
VISION = "Qwen/Qwen3-VL-30B-A3B-Instruct"
WRITER = "openai/gpt-oss-120b" # on the free tier
EDIT = "black-forest-labs/FLUX-2-klein-4b" # keeps the product, changes the scene
VIDEO = "kuaishou/kling-v2.6" # start image to 5 s of motion
TTS, VOICE = "hexgrad/Kokoro-82M", "af_bella"
ledger = []
OUT = Path("out"); OUT.mkdir(exist_ok=True)
def log(step, model, t0, cost):
ledger.append({"step": step, "model": model, "seconds": round(time.time() - t0, 2), "cost_usd": cost})
print(f"{step:<10} {model:<40} {time.time() - t0:7.2f}s ${cost:.6f}")
def data_url(path: Path) -> str:
b = path.read_bytes() # models answer jpeg, png or webp whatever the file is called: sniff it
mime = "image/png" if b[:4] == b"\x89PNG" else "image/webp" if b[8:12] == b"WEBP" else "image/jpeg"
return f"data:{mime};base64,{base64.b64encode(b).decode()}"
def json_call(model, messages, schema, name):
raw = client.chat.completions.with_raw_response.create(
model=model, messages=messages, temperature=0.4,
response_format={"type": "json_schema", "json_schema": {"name": name, "strict": True, "schema": schema}})
return json.loads(raw.parse().choices[0].message.content), float(raw.headers.get("x-llm-cost", 0))
def save_image(item: dict, path: Path) -> None:
url = item.get("url") or ""
if url.startswith("data:"):
path.write_bytes(base64.b64decode(url.split(",", 1)[1]))
elif url:
path.write_bytes(requests.get(url, timeout=60).content)
else:
path.write_bytes(base64.b64decode(item["b64_json"]))
def video(body: dict, path: Path) -> float:
"""Submit, poll with backoff (riding out 429s and 5xx), download. Returns the billed cost."""
h = {"Authorization": f"Bearer {KEY}"}
for attempt in range(5):
res = requests.post(f"{API}/videos", json=body, headers=h, timeout=60)
if res.status_code not in (429, 500, 502, 503, 504):
break
time.sleep(int(res.headers.get("Retry-After", 2 ** attempt)))
res.raise_for_status()
job, cost = res.json(), float(res.headers.get("x-llm-cost", 0))
wait = 5
while True:
time.sleep(wait)
r = requests.get(f"{API}/videos/{job['id']}", headers=h, timeout=30)
if r.status_code in (429, 500, 502, 503, 504):
wait = min(wait * 2, 30); continue
r.raise_for_status()
job = r.json()
if job["status"] == "completed":
break
if job["status"] == "failed":
raise RuntimeError(f"video {job['id']} failed: {job.get('error')}")
wait = min(wait * 1.5, 20)
path.write_bytes(requests.get(f"{API}/videos/{job['id']}/download", headers=h, timeout=300).content)
return cost
def main(photo: str) -> None:
photo = Path(photo)
t0 = time.time()
facts, cost = json_call(VISION, [{"role": "user", "content": [
{"type": "text", "text": "What is this product? Name it plainly, list what it is made of and who it is for. Only what you can see."},
{"type": "image_url", "image_url": {"url": data_url(photo)}}]}],
{"type": "object", "additionalProperties": False, "required": ["product", "materials", "audience"],
"properties": {"product": {"type": "string"}, "materials": {"type": "array", "items": {"type": "string"}}, "audience": {"type": "string"}}},
"facts")
log("vision", VISION, t0, cost)
t0 = time.time()
script, cost = json_call(WRITER, [{"role": "user", "content":
f"Write a 7-second video ad for: {json.dumps(facts)}. Two scenes. Scene 1 is 5 seconds of motion, scene 2 a 2-second end card. "
"Narration: one sentence of at most 16 words, spoken over both scenes. Edit prompts describe a new setting for the same product, "
"keeping the product exactly as photographed. The motion prompt describes camera movement only."}],
{"type": "object", "additionalProperties": False, "required": ["tagline", "narration", "scene1_edit", "scene1_motion", "scene2_edit"],
"properties": {k: {"type": "string"} for k in ["tagline", "narration", "scene1_edit", "scene1_motion", "scene2_edit"]}},
"script")
log("script", WRITER, t0, cost)
(OUT / "script.json").write_text(json.dumps({"facts": facts, "script": script}, indent=2))
stills = {}
# The end card carries the tagline as type in the image: no font setup, and ffmpeg builds without drawtext still work.
card = f'{script["scene2_edit"]} Leave clear space at the bottom with the words "{script["tagline"]}" in clean white sans-serif type.'
for scene, prompt in [("scene1", script["scene1_edit"]), ("scene2", card)]:
t0 = time.time()
raw = client.images.with_raw_response.generate(model=EDIT, prompt=prompt, extra_body={"images": [data_url(photo)], "aspect_ratio": "16:9"})
stills[scene] = OUT / f"{scene}.jpg"
save_image(raw.parse().data[0].model_dump(), stills[scene])
log(f"edit {scene}", EDIT, t0, float(raw.headers.get("x-llm-cost", 0)))
t0 = time.time()
if (OUT / "scene1.mp4").exists(): # a rerun after a later step failed does not pay for the video again
cost = 0.0
else:
cost = video({"model": VIDEO, "prompt": script["scene1_motion"], "duration": 5, "generate_audio": False,
"start_image": data_url(stills["scene1"])}, OUT / "scene1.mp4")
log("video", VIDEO, t0, cost)
t0 = time.time()
raw = client.audio.speech.with_raw_response.create(model=TTS, voice=VOICE, input=script["narration"], response_format="wav")
(OUT / "narration.wav").write_bytes(raw.content)
log("voice", TTS, t0, float(raw.headers.get("x-llm-cost", 0)))
# Scene 1 (5 s of motion) + scene 2 (the 2 s end card), narration on top, 1280x720 H.264.
t0 = time.time()
subprocess.run([
"ffmpeg", "-y", "-loglevel", "error",
"-i", str(OUT / "scene1.mp4"), "-loop", "1", "-t", "2", "-i", str(stills["scene2"]), "-i", str(OUT / "narration.wav"),
"-filter_complex",
"[0:v]scale=1280:720:force_original_aspect_ratio=increase,crop=1280:720,fps=30,setsar=1[a];"
"[1:v]scale=1280:720:force_original_aspect_ratio=increase,crop=1280:720,fps=30,setsar=1[b];"
"[a][b]concat=n=2:v=1:a=0[v];[2:a]apad[n]",
"-map", "[v]", "-map", "[n]", "-t", "7", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", str(OUT / "ad.mp4"),
], check=True)
log("mux", "ffmpeg", t0, 0.0)
total = sum(s["cost_usd"] for s in ledger)
Path("run.json").write_text(json.dumps({"steps": ledger, "total_usd": total}, indent=2))
print(f"\nout/ad.mp4 total ${total:.4f} narration: {script['narration']}")
if __name__ == "__main__":
main(sys.argv[1])
// One product photo to a narrated 7-second video ad: vision, script, scene edits, image-to-video, voice, mux.
//
// npm install openai (and ffmpeg on PATH)
// RUNCRATE_API_KEY=... npx tsx product-photo-to-video-ad.ts product.jpg
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import OpenAI from "openai";
const API = "https://api.runcrate.ai/v1";
const KEY = process.env.RUNCRATE_API_KEY ?? "";
const client = new OpenAI({ baseURL: API, apiKey: KEY, maxRetries: 4, timeout: 180_000 });
const VISION = "Qwen/Qwen3-VL-30B-A3B-Instruct";
const WRITER = "openai/gpt-oss-120b"; // on the free tier
const EDIT = "black-forest-labs/FLUX-2-klein-4b"; // keeps the product, changes the scene
const VIDEO = "kuaishou/kling-v2.6"; // start image to 5 s of motion
const TTS = "hexgrad/Kokoro-82M";
const VOICE = "af_bella";
const ledger: { step: string; model: string; seconds: number; cost_usd: number }[] = [];
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
mkdirSync("out", { recursive: true });
function log(step: string, model: string, t0: number, cost: number) {
const seconds = (Date.now() - t0) / 1000;
ledger.push({ step, model, seconds, cost_usd: cost });
console.log(`${step.padEnd(10)} ${model.padEnd(40)} ${seconds.toFixed(2).padStart(7)}s $${cost.toFixed(6)}`);
}
function dataUrl(path: string): string {
const b = readFileSync(path); // models answer jpeg, png or webp whatever the file is called: sniff it
const mime = b.subarray(0, 4).toString("latin1") === "\x89PNG" ? "image/png" : b.subarray(8, 12).toString() === "WEBP" ? "image/webp" : "image/jpeg";
return `data:${mime};base64,${b.toString("base64")}`;
}
async function jsonCall<T>(model: string, content: OpenAI.Chat.ChatCompletionUserMessageParam["content"], schema: Record<string, unknown>, name: string): Promise<[T, number]> {
const { data, response } = await client.chat.completions
.create({ model, messages: [{ role: "user", content }], temperature: 0.4, response_format: { type: "json_schema", json_schema: { name, strict: true, schema } } })
.withResponse();
return [JSON.parse(data.choices[0].message.content ?? "{}"), Number(response.headers.get("x-llm-cost") ?? 0)];
}
async function saveImage(url: string, path: string) {
writeFileSync(path, url.startsWith("data:") ? Buffer.from(url.split(",", 2)[1], "base64") : Buffer.from(await (await fetch(url)).arrayBuffer()));
}
/** Submit, poll with backoff (riding out 429s and 5xx), download. Returns the billed cost. */
async function video(body: object, path: string): Promise<number> {
const h = { Authorization: `Bearer ${KEY}`, "Content-Type": "application/json" };
let res = await fetch(`${API}/videos`, { method: "POST", headers: h, body: JSON.stringify(body) });
for (let attempt = 0; [429, 500, 502, 503, 504].includes(res.status) && attempt < 5; attempt++) {
await sleep(Number(res.headers.get("retry-after") ?? 2 ** attempt) * 1000);
res = await fetch(`${API}/videos`, { method: "POST", headers: h, body: JSON.stringify(body) });
}
if (!res.ok) throw new Error(`video submit answered ${res.status}: ${await res.text()}`);
const cost = Number(res.headers.get("x-llm-cost") ?? 0);
let job = (await res.json()) as { id: string; status: string; error?: unknown };
let wait = 5000;
for (;;) {
await sleep(wait);
const r = await fetch(`${API}/videos/${job.id}`, { headers: h });
if ([429, 500, 502, 503, 504].includes(r.status)) {
wait = Math.min(wait * 2, 30_000);
continue;
}
if (!r.ok) throw new Error(`video poll answered ${r.status}`);
job = (await r.json()) as typeof job;
if (job.status === "completed") break;
if (job.status === "failed") throw new Error(`video ${job.id} failed: ${JSON.stringify(job.error)}`);
wait = Math.min(wait * 1.5, 20_000);
}
const file = await fetch(`${API}/videos/${job.id}/download`, { headers: h });
writeFileSync(path, Buffer.from(await file.arrayBuffer()));
return cost;
}
const photo = process.argv[2];
let t0 = Date.now();
const [facts, visionCost] = await jsonCall<{ product: string; materials: string[]; audience: string }>(
VISION,
[
{ type: "text", text: "What is this product? Name it plainly, list what it is made of and who it is for. Only what you can see." },
{ type: "image_url", image_url: { url: dataUrl(photo) } },
],
{
type: "object",
additionalProperties: false,
required: ["product", "materials", "audience"],
properties: { product: { type: "string" }, materials: { type: "array", items: { type: "string" } }, audience: { type: "string" } },
},
"facts",
);
log("vision", VISION, t0, visionCost);
t0 = Date.now();
const keys = ["tagline", "narration", "scene1_edit", "scene1_motion", "scene2_edit"] as const;
const [script, scriptCost] = await jsonCall<Record<(typeof keys)[number], string>>(
WRITER,
`Write a 7-second video ad for: ${JSON.stringify(facts)}. Two scenes. Scene 1 is 5 seconds of motion, scene 2 a 2-second end card. ` +
"Narration: one sentence of at most 16 words, spoken over both scenes. Edit prompts describe a new setting for the same product, " +
"keeping the product exactly as photographed. The motion prompt describes camera movement only.",
{ type: "object", additionalProperties: false, required: keys, properties: Object.fromEntries(keys.map((k) => [k, { type: "string" }])) },
"script",
);
log("script", WRITER, t0, scriptCost);
writeFileSync("out/script.json", JSON.stringify({ facts, script }, null, 2));
// The end card carries the tagline as type in the image: no font setup, and ffmpeg builds without drawtext still work.
const card = `${script.scene2_edit} Leave clear space at the bottom with the words "${script.tagline}" in clean white sans-serif type.`;
for (const [scene, prompt] of [["scene1", script.scene1_edit], ["scene2", card]]) {
t0 = Date.now();
const { data, response } = await client.images
.generate({ model: EDIT, prompt, images: [dataUrl(photo)], aspect_ratio: "16:9" } as OpenAI.Images.ImageGenerateParamsNonStreaming)
.withResponse();
await saveImage(data.data?.[0]?.url ?? "", `out/${scene}.jpg`);
log(`edit ${scene}`, EDIT, t0, Number(response.headers.get("x-llm-cost") ?? 0));
}
t0 = Date.now();
// A rerun after a later step failed does not pay for the video again.
const videoCost = existsSync("out/scene1.mp4")
? 0
: await video({ model: VIDEO, prompt: script.scene1_motion, duration: 5, generate_audio: false, start_image: dataUrl("out/scene1.jpg") }, "out/scene1.mp4");
log("video", VIDEO, t0, videoCost);
t0 = Date.now();
const speech = await client.audio.speech.create({ model: TTS, voice: VOICE, input: script.narration, response_format: "wav" }).withResponse();
writeFileSync("out/narration.wav", Buffer.from(await speech.data.arrayBuffer()));
log("voice", TTS, t0, Number(speech.response.headers.get("x-llm-cost") ?? 0));
// Scene 1 (5 s of motion) + scene 2 (the 2 s end card), narration on top, 1280x720 H.264.
t0 = Date.now();
execFileSync("ffmpeg", [
"-y", "-loglevel", "error",
"-i", "out/scene1.mp4", "-loop", "1", "-t", "2", "-i", "out/scene2.jpg", "-i", "out/narration.wav",
"-filter_complex",
"[0:v]scale=1280:720:force_original_aspect_ratio=increase,crop=1280:720,fps=30,setsar=1[a];" +
"[1:v]scale=1280:720:force_original_aspect_ratio=increase,crop=1280:720,fps=30,setsar=1[b];" +
"[a][b]concat=n=2:v=1:a=0[v];[2:a]apad[n]",
"-map", "[v]", "-map", "[n]", "-t", "7", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "out/ad.mp4",
]);
log("mux", "ffmpeg", t0, 0);
const total = ledger.reduce((s, r) => s + r.cost_usd, 0);
writeFileSync("run.json", JSON.stringify({ steps: ledger, total_usd: total }, null, 2));
console.log(`\nout/ad.mp4 total $${total.toFixed(4)} narration: ${script.narration}`);
What the proof run cost
The Python file on a studio photo of a glass bottle. Measured on 2026-10-23 againstapi.runcrate.ai; costs are what the account was billed, read from each response’s X-LLM-Cost header or usage.cost.
| Step | Model or tool | Time | Cost |
|---|---|---|---|
| Product facts | Qwen/Qwen3-VL-30B-A3B-Instruct | 1.8 s | $0.0002 |
| Script | openai/gpt-oss-120b (free tier) | 1.8 s | $0.0000 |
| Two scene edits | black-forest-labs/FLUX-2-klein-4b | 3.9 s | $0.0022 |
| 5 s of motion, no audio | kuaishou/kling-v2.6 | 145.0 s | $0.3675 |
| Narration | hexgrad/Kokoro-82M | 1.4 s | $0.0053 |
| Mux 1280x720 | ffmpeg | 0.5 s | $0 |
| Total | about 155 s | $0.3752 |
drawtext; the rerun reused the paid video and cost $0.0090 more. The TypeScript file end to end: $0.3735 in 217 s.
Scale it
- A catalog: one ad per product is independent work, so run products in parallel (a thread pool locally, or a job group with one product per
--paramsentry). - Variants for testing: the scene edits cost a tenth of a cent each, so make five end cards and pick; only the video costs real money.
- The step that costs is cached: a rerun after a later failure does not pay for the video again (the script skips it when
out/scene1.mp4exists). - Ten-second ads:
duration: 10on Kling v2.6 doubles the video cost, andgenerate_audio: truedoubles it again.
Gotchas found in the proof run
black-forest-labs/FLUX.1-kontext-devhung for over 15 minutes on the first edit on the proof day, and a direct retry timed out at 180 s. FLUX.2 klein took 1.6 to 3.5 s for the same edit, with the reference passed asimages: [data_url].- Image models answer JPEG, PNG or WebP whatever you name the file. The script sniffs the bytes before it sends a data URL, so a WebP saved as
.jpgis still labelled right. - Homebrew’s ffmpeg has no
drawtext, so a tagline drawn by ffmpeg fails there. Asking the image model for the words in the end card worked first time. - Kling v2.6 bills per second of the length you ask for, at $0.0735 a second without audio (catalog price on the day).