music-video-gen/flow-state/src/look/shots.js

// Shot planning: the level of hierarchy between a song section and a frame.
//
// A section is a STAGE of the song (intro, build, drop, …) and can easily run
// ninety seconds. One scene held for ninety seconds reads as a still image with
// a wobble on it, no matter how much per-frame reactivity is underneath. So a
// section is cut into SHOTS, each showing one of the section's few "stage
// visuals" — the roster the look generator picked for that kind of section.
//
// Two rules keep this from turning into a shuffle:
//
//   * the roster is per section KIND, not per section, so all of a track's drops
//     still cut between the same two or three visuals and the video keeps an
//     identity;
//   * cuts land on phrase lines, so a change of image lands with the music
//     instead of across it.
//
// Shot length follows energy: a drop cuts every four to eight bars, an intro
// holds for eight to sixteen, and nothing holds past the ceiling below.
// Everything here is seeded, so a track always cuts in the same places.

/** Never cut faster than this, whatever the tempo or the energy says. */
export const MIN_SHOT_SECONDS = 5;

/**
 * And never hold longer than this either. Half a minute of one image is the
 * complaint this whole level of hierarchy exists to answer, so it is a hard
 * ceiling rather than something the bar maths is trusted to stay under: at a
 * slow tempo sixteen bars is already past it.
 */
export const MAX_SHOT_SECONDS = 22;

/** Below this section energy, a shot change is always a dissolve, never a cut. */
export const HARD_CUT_ENERGY = 0.66;

/**
 * The section's cutting RHYTHM, in bars per shot.
 *
 * This used to be a single bar count, and the section was then divided into
 * that many equal pieces. Measured, a five-minute track at 90 BPM came out as
 * sixteen shots of 19.3, 18.7, 18.7, 18.7 … 18.7, 19.3 seconds — a metronome.
 * Every cut landing on the same pulse for five minutes is the most fatiguing
 * edit rhythm available, and no amount of variety in what the shots CONTAIN
 * fixes it, because the fatigue is in the timing rather than in the images.
 *
 * So a section carries a repeating PATTERN instead: a long hold, two quick
 * ones, a long hold. The pattern is walked in order and repeats, which is what
 * makes it read as phrasing rather than as randomness — random shot lengths
 * would satisfy any "lengths must vary" test and would look worse, because the
 * ear is following an eight-bar structure and the eye would not be.
 *
 * Every entry is a power-of-two bar count, so a cut is always on a phrase line
 * of some depth even before it is snapped to a downbeat.
 */
function rhythmFor(energy, rng, story = null) {
    // Where the section sits in the story shifts which band it cuts in. A song
    // does not only get louder toward its climax, it gets more urgent, and edit
    // rate is the one register that says urgency without changing the image at
    // all. The resolution goes the other way and holds — the last thing a video
    // should do is keep cutting at the pace of the thing that just ended.
    if (story) {
        const urgency = (story.tension - 0.5) * 0.22;
        energy = Math.max(0, Math.min(1, energy + urgency));
        if (story.act === 'resolution') energy = Math.min(energy, 0.44);
    }
    if (energy > 0.72) {
        // Loud material: quick cuts, but still answered by a longer hold.
        return rng.pick([[4, 4, 8], [8, 4, 4], [4, 4, 4, 8], [8, 8, 4, 4], [4, 8, 4, 4]]);
    }
    if (energy > 0.45) {
        return rng.pick([[8, 8, 16], [16, 8, 8], [8, 16, 8], [8, 8, 8, 16], [16, 8, 16, 8]]);
    }
    // Quiet material holds, and departs from the hold rather than the reverse.
    return rng.pick([[16, 16, 8], [16, 8, 16], [16, 16, 16, 8], [8, 16, 16]]);
}

/**
 * Fit a bar pattern to real seconds.
 *
 * The ceiling is applied by scaling the WHOLE pattern rather than by clamping
 * each entry, because clamping destroys exactly what the pattern is for: at 90
 * BPM a 16-bar hold is 42 s and a 4-bar one is 10 s, and clamping both to 22
 * turns 16/16/8 into a metronome again. Scaling keeps the 2:1 relationships
 * that make the rhythm legible. Only after that is each entry clamped, to catch
 * whatever the scale could not reconcile.
 */
function fitPattern(pattern, barSeconds) {
    const unit = barSeconds > 0.2 ? barSeconds : 3;
    const raw = pattern.map((bars) => bars * unit);

    const hi = Math.max(...raw);
    const scale = hi > MAX_SHOT_SECONDS ? MAX_SHOT_SECONDS / hi : 1;

    return raw.map((s) => Math.min(MAX_SHOT_SECONDS, Math.max(MIN_SHOT_SECONDS, s * scale)));
}

/**
 * The downbeat nearest `time` that also makes a LEGAL shot when measured from
 * `from`, or null if there isn't one within tolerance.
 *
 * The legality bound is the point. Snapping to the merely-nearest downbeat can
 * push a cut later than the ideal, and a 22-second shot snapped 0.75s late is a
 * 22.75-second shot — over the ceiling the pattern was fitted to respect. So the
 * search is restricted to downbeats that keep the shot inside the floor and the
 * ceiling, which usually means taking the downbeat just before the ideal rather
 * than the one just after. Landing on the grid matters more than landing on the
 * closest line.
 */
function snapCut(from, ideal, downbeats, tolerance) {
    let best = null;
    let bestDist = Infinity;
    for (const d of downbeats) {
        if (d - from < MIN_SHOT_SECONDS) continue;
        if (d - from > MAX_SHOT_SECONDS) break;        // sorted: only gets worse
        const dist = Math.abs(d - ideal);
        if (dist < bestDist) { bestDist = dist; best = d; }
        else if (d > ideal) break;                     // past the minimum
    }
    return best !== null && bestDist <= tolerance ? best : null;
}

/**
 * Divide a section into shots.
 *
 * @param {object} section    a track section (start/end/startFrame/endFrame)
 * @param {object} track      FeatureTrack, for fps and the bar grid
 * @param {object} bias       the section's bias, for energy
 * @param {number} variantCount how many stage visuals the section has
 * @param {Rng} rng
 * @param {object|null} story  this section's story state, see look/Story.js
 * @returns {Array<{index,startFrame,endFrame,variant,hardCut}>}
 */
export function planShots(section, track, bias, variantCount, rng, story = null) {
    const fps = track.fps;
    const duration = Math.max(0, section.end - section.start);
    const barSeconds = (track.tempo.period * track.tempo.beatsPerBar) / fps;

    const lengths = fitPattern(rhythmFor(bias.energy, rng, story), barSeconds);
    const shortest = Math.min(...lengths);

    // A section with only one visual to show has nothing to cut TO. It still
    // gets shot boundaries: a boundary changes the FRAMING as well as the
    // image, so the same visual filmed again at another size is a shot, and the
    // ceiling is about how long one image is held rather than about how many
    // images a section has.
    //
    // This used to skip cut planning entirely, and the exemption swallowed the
    // ceiling with it — measured, a 150 BPM track held one visual for 151.98
    // seconds, seven times the limit, which is exactly the complaint this whole
    // level of hierarchy exists to answer. What it does still suppress is the
    // hard cut: cutting straight between two framings of one image is a jump
    // cut, so those boundaries always dissolve.
    const single = variantCount < 2;

    // Walk the pattern, laying shots end to end from the section start. Each cut
    // is then pulled onto the nearest downbeat; the tolerance stays under half
    // the shortest shot so a snap can never reorder two cuts or collapse one
    // onto another. Drift from snapping does not accumulate, because the next
    // shot is measured from the snapped time rather than from the ideal one.
    const tolerance = Math.min(barSeconds * 1.5, shortest * 0.35);
    const downbeats = track.tempo.downbeats || [];
    const cuts = [];

    {
        let at = section.start;
        for (let k = 0; k < 512; k++) {
            const raw = at + lengths[k % lengths.length];
            const snapped = snapCut(at, raw, downbeats, tolerance);
            const time = snapped !== null ? snapped : raw;

            // Stop when the remainder would be shorter than a legal shot: the
            // tail belongs to the shot already running rather than becoming a
            // stub. This is also what ends the loop on any section length.
            if (section.end - time < MIN_SHOT_SECONDS) break;
            cuts.push(time);
            at = time;
        }

        // Absorbing the tail can push the closing shot past the ceiling — a
        // 22-second shot plus a 4-second remainder is 26. Split it. The span is
        // over MAX by construction here, and MAX is more than twice MIN, so both
        // halves are legal shots.
        const lastCut = cuts.length ? cuts[cuts.length - 1] : section.start;
        if (section.end - lastCut > MAX_SHOT_SECONDS) {
            cuts.push(lastCut + (section.end - lastCut) / 2);
        }
    }

    const bounds = [section.start, ...cuts, section.end];
    const shots = [];
    const lastSeen = new Array(variantCount).fill(-1);
    let previousVariant = -1;
    for (let i = 0; i < bounds.length - 1; i++) {
        const variant = i === 0 ? 0 : pickVariant(variantCount, previousVariant, lastSeen, i, rng);
        lastSeen[variant] = i;
        previousVariant = variant;
        shots.push({
            index: i,
            startFrame: i === 0 ? section.startFrame : Math.round(bounds[i] * fps),
            endFrame: i === bounds.length - 2 ? section.endFrame : Math.round(bounds[i + 1] * fps),
            start: bounds[i],
            end: bounds[i + 1],
            variant,
            // A dissolve is the default. A straight cut is what makes a drop feel
            // edited, but on anything calmer it reads as a glitch, so cuts are
            // gated on real energy rather than sprinkled everywhere: nothing below
            // the threshold ever cuts, and only the loudest material cuts often.
            // A single-visual section changes framing, not image. Cutting hard
            // between two framings of the same thing is a jump cut, so those
            // boundaries always dissolve.
            hardCut: !single && bias.energy > HARD_CUT_ENERGY
                && rng.bool(Math.min(0.85, (bias.energy - HARD_CUT_ENERGY) * 2.5)),
        });
    }
    return shots;
}

/**
 * Next visual in the rotation.
 *
 * The shape is A B A C A D: the anchor comes back between companions, so the
 * section reads as one idea with departures from it rather than as a playlist.
 * It is a strong tendency and not a rule — strict alternation is audible as a
 * pattern within about three cycles.
 *
 * When a companion is due, the LEAST RECENTLY SHOWN one wins. With a roster of
 * four that is the difference between a section showing B, C, D and a section
 * showing B twice and never reaching D.
 */
function pickVariant(variantCount, previous, lastSeen, shotIndex, rng) {
    if (previous !== 0 && rng.bool(0.75)) return 0;

    // A companion this section has not shown yet wins outright. Weighting it
    // heavily was not enough — measured, a five-shot section still came out
    // 0,2,0,2,0 about a fifth of the time, so the roster existed and the shots
    // never reached it. Which unseen one is still a free choice, so the order
    // varies between sections; only the coverage is guaranteed.
    const unseen = [];
    for (let v = 1; v < variantCount; v++) {
        if (v !== previous && lastSeen[v] < 0) unseen.push(v);
    }
    if (unseen.length) return rng.pick(unseen);

    const options = [];
    const weights = [];
    for (let v = 0; v < variantCount; v++) {
        if (v === previous) continue;
        options.push(v);
        // Everything has been shown at least once: fall back to least recently
        // seen, with the anchor kept in the draw so the rotation cannot become
        // a rigid cycle.
        weights.push(v === 0 ? 1 : 2 + (shotIndex - lastSeen[v]));
    }
    if (!options.length) return 0;
    return rng.pickWeighted(options, weights);
}