gerbeur/api/lib/ffmpeg.ts

/**
 * Makes what the server derives from an upload with ffmpeg: a video's still,
 * which the thumbnails route serves, an audio file's waveform, which the
 * waveform route serves, and the scaled-down copies of any image, which
 * `serveImage` (lib/upload.ts) serves for every route that answers one. Each
 * is made on its first request and kept under a name this module gives, a
 * dump's in `THUMBNAILS_DIR` and the scaled copies in `SCALED_DIR`, until
 * dump-service replaces or deletes the upload and
 * {@link removeGeneratedMedia} drops them. {@link generatedMediaSweepJob}
 * takes what that leaves behind.
 *
 * @module
 */

import { dirname, relative } from "node:path";
import {
  DUMPS_DIR,
  SCALED_DIR,
  THUMBNAILS_DIR,
  UPLOADS_DIR,
} from "../config.ts";
import { IMAGE_WIDTHS, type ImageWidth } from "../../shared/image.ts";
import type { Job } from "./scheduler.ts";
import { type DumpWaveform, peaksOf } from "../../shared/waveform.ts";
import type { APIResponse } from "../../shared/api.ts";

/**
 * Spares a process per uncached request: whether ffmpeg runs is probed once
 * per process, and kept here.
 */
let ffmpegProbe: Promise<boolean> | undefined;

function ffmpegAvailable(): Promise<boolean> {
  ffmpegProbe ??= (async () => {
    try {
      const { success } = await new Deno.Command("ffmpeg", {
        args: ["-version"],
        stderr: "null",
        stdout: "null",
      }).output();
      return success;
    } catch {
      return false;
    }
  })();
  return ffmpegProbe;
}

/**
 * Lists the container formats ffmpeg may read a video as. The upload's type is
 * only what the uploader claimed, and ffmpeg guesses the format from the bytes,
 * so a "video" could be an HLS or concat playlist pointing it at local files or
 * internal URLs. With this list, and `-protocol_whitelist file`, ffmpeg reads
 * nothing but the file, and only as a video.
 */
const VIDEO_FORMATS =
  "mov,mp4,m4a,3gp,3g2,mj2,matroska,webm,avi,flv,mpegts,mpeg,ogg,asf,h264,hevc";
/**
 * Lists the formats ffmpeg may read an image as, for the reason given at
 * {@link VIDEO_FORMATS}.
 */
const IMAGE_FORMATS = "jpeg_pipe,png_pipe,webp_pipe,bmp_pipe,gif";
/**
 * Lists the formats ffmpeg may read an audio file as, for the reason given at
 * {@link VIDEO_FORMATS}; an `.m4a` or a `.webm` is read by its container's.
 */
const AUDIO_FORMATS =
  "mp3,ogg,flac,wav,aiff,aac,asf,caf,w64,wv,ape,mov,mp4,m4a,matroska,webm";

/**
 * Returns where the still and the waveform generated from dump `dumpId`'s
 * upload are kept.
 */
export function generatedPaths(dumpId: string): {
  still: string;
  waveform: string;
} {
  const base = `${THUMBNAILS_DIR}/${dumpId}`;
  return { still: `${base}.jpg`, waveform: `${base}.waveform.json` };
}

/**
 * The formats a scaled copy comes in: WebP for a browser, and a JPEG or a PNG
 * for what doesn't read it, as some mail clients don't.
 */
export type ScaledFormat = "webp" | "jpg" | "png";
const SCALED_FORMATS: readonly ScaledFormat[] = ["webp", "jpg", "png"];

/**
 * Returns where the copy of the upload at `srcPath` scaled to `width` is
 * kept, in `format`: under `SCALED_DIR`, at the upload's own path below
 * `UPLOADS_DIR`.
 */
export function scaledImagePath(
  srcPath: string,
  width: ImageWidth,
  format: ScaledFormat = "webp",
): string {
  return `${SCALED_DIR}/${relative(UPLOADS_DIR, srcPath)}.w${width}.${format}`;
}

/**
 * Deletes the scaled copies of the upload at `srcPath`, once it is deleted.
 * One that is replaced needs no call: its copies are made again, being older
 * than it ({@link ensureGenerated}).
 */
export async function removeScaledImages(srcPath: string): Promise<void> {
  await Promise.all(
    IMAGE_WIDTHS.flatMap((width) =>
      SCALED_FORMATS.map((format) =>
        Deno.remove(scaledImagePath(srcPath, width, format)).catch(() => {})
      )
    ),
  );
}

/**
 * Deletes what was generated from dump `dumpId`'s upload, once dump-service
 * has replaced or deleted it. The routes make them again from a new one.
 */
export async function removeGeneratedMedia(dumpId: string): Promise<void> {
  const { still, waveform } = generatedPaths(dumpId);
  await Promise.all([
    ...[still, waveform].map((path) => Deno.remove(path).catch(() => {})),
    removeScaledImages(`${DUMPS_DIR}/${dumpId}`),
  ]);
}

/**
 * Writes `outPath` with `write`, which gets a path beside it to fill, renamed
 * over `outPath` once done, so a concurrent reader never gets half a file.
 */
async function writeAtomically(
  outPath: string,
  write: (tmpPath: string) => Promise<void>,
): Promise<void> {
  await Deno.mkdir(dirname(outPath), { recursive: true });
  const tmpPath = `${outPath}.${crypto.randomUUID()}.tmp`;
  try {
    await write(tmpPath);
    await Deno.rename(tmpPath, outPath);
  } catch (err) {
    await Deno.remove(tmpPath).catch(() => {});
    throw err;
  }
}

/**
 * Runs ffmpeg with `args`, writing to `outPath`.
 *
 * @throws {Error} When ffmpeg fails.
 */
function runFfmpeg(args: string[], outPath: string): Promise<void> {
  return writeAtomically(outPath, async (tmpPath) => {
    const { success } = await new Deno.Command("ffmpeg", {
      args: ["-protocol_whitelist", "file", ...args, "-y", tmpPath],
      stdout: "null",
      stderr: "null",
    }).output();
    if (!success) throw new Error("ffmpeg failed");
  });
}

/** Writes a 320-pixel-wide JPEG still of a video, one second in. */
export function videoStill(srcPath: string, outPath: string): Promise<void> {
  return runFfmpeg([
    "-format_whitelist",
    VIDEO_FORMATS,
    "-ss",
    "00:00:01",
    "-i",
    srcPath,
    "-frames:v",
    "1",
    "-vf",
    "scale=320:-1",
    "-f",
    "image2",
    "-c:v",
    "mjpeg",
  ], outPath);
}

/**
 * Maps a JPEG's EXIF orientation to the filters that turn its pixels upright.
 * A browser applies the orientation to the upload itself; the copy carries no
 * EXIF, so it is turned here, and not by ffmpeg's own autorotation, which
 * only its recent versions apply to images.
 */
const ORIENTATION_FILTERS: Record<number, string> = {
  2: "hflip",
  3: "hflip,vflip",
  4: "vflip",
  5: "transpose=0",
  6: "transpose=1",
  7: "transpose=3",
  8: "transpose=2",
};

/** What ffmpeg is told to write each {@link ScaledFormat} with. */
const ENCODERS: Record<ScaledFormat, string[]> = {
  webp: ["-c:v", "libwebp", "-q:v", "80", "-loop", "0", "-f", "webp"],
  jpg: ["-c:v", "mjpeg", "-q:v", "4", "-f", "image2"],
  png: ["-c:v", "png", "-f", "image2"],
};

/**
 * What {@link scaleImage} is told of the image it scales, which
 * `scalableImage` (lib/upload.ts) reads from its first bytes.
 */
export interface ImageTraits {
  /** The EXIF orientation to turn the image by, 1 when it is upright. */
  orientation: number;
  /**
   * Whether its frames are an animation, as a GIF's are. A still may hold
   * more than one image too, a phone's JPEG carrying a gain map or a depth
   * map after the photo, and ffmpeg reads each as a frame.
   */
  animated: boolean;
}

/**
 * Writes a copy of an image at most `width` wide, never wider than the image
 * itself, as a WebP unless `format` names another. WebP keeps a PNG's
 * transparency and a GIF's animation, at a fraction of either's bytes.
 */
export function scaleImage(
  srcPath: string,
  outPath: string,
  width: ImageWidth,
  { orientation, animated }: ImageTraits,
  format: ScaledFormat = "webp",
): Promise<void> {
  const upright = ORIENTATION_FILTERS[orientation];
  // A JPEG's colour planes halve its height, which they need even.
  const height = format === "jpg" ? -2 : -1;
  // Only an animation keeps its frames, and only as a WebP: of anything else
  // the first is the image, and a copy of them all would flash through what
  // follows it.
  const frames = animated && format === "webp" ? [] : ["-frames:v", "1"];
  return runFfmpeg([
    "-noautorotate",
    "-format_whitelist",
    IMAGE_FORMATS,
    "-i",
    srcPath,
    "-vf",
    `${upright ? `${upright},` : ""}scale='min(${width},iw)':${height}`,
    "-an",
    ...frames,
    ...ENCODERS[format],
  ], outPath);
}

/**
 * Returns `bytes`, an image held in memory, scaled as {@link scaleImage}
 * does, or null when it can't be, because ffmpeg is missing or failed. The
 * image proxy scales what it fetched with it, and keeps the result itself.
 */
export async function scaleImageBytes(
  bytes: Uint8Array,
  width: ImageWidth,
  traits: ImageTraits,
): Promise<Uint8Array | null> {
  if (!await ffmpegAvailable()) return null;
  const srcPath = `${SCALED_DIR}/tmp/${crypto.randomUUID()}`;
  const outPath = `${srcPath}.webp`;
  try {
    await Deno.mkdir(dirname(srcPath), { recursive: true });
    await Deno.writeFile(srcPath, bytes);
    await scaleImage(srcPath, outPath, width, traits);
    return await Deno.readFile(outPath);
  } catch {
    return null;
  } finally {
    await Promise.all(
      [srcPath, outPath].map((path) => Deno.remove(path).catch(() => {})),
    );
  }
}

/**
 * The sample rate the audio is decoded at for its waveform. A bar stands for
 * a second or more of a song, so 8 kHz keeps every peak worth drawing at a
 * sixth of the samples.
 */
const PEAKS_SAMPLE_RATE = 8000;
/** How many samples each running peak covers before `peaksOf` buckets them. */
const PEAKS_WINDOW = 256;

/**
 * Decodes the audio of `srcPath` to its waveform's peaks. ffmpeg streams the
 * samples, mono at {@link PEAKS_SAMPLE_RATE}, and only the peak of each
 * {@link PEAKS_WINDOW} is kept, so an hour of audio holds a few hundred
 * kilobytes at most.
 *
 * @throws {Error} When ffmpeg fails or the file holds no audio.
 */
async function audioPeaks(srcPath: string): Promise<number[]> {
  const child = new Deno.Command("ffmpeg", {
    args: [
      "-protocol_whitelist",
      "file",
      "-format_whitelist",
      AUDIO_FORMATS,
      "-i",
      srcPath,
      "-vn",
      "-ac",
      "1",
      "-ar",
      String(PEAKS_SAMPLE_RATE),
      "-f",
      "s16le",
      "-",
    ],
    stdin: "null",
    stdout: "piped",
    stderr: "null",
  }).spawn();

  const windows: number[] = [];
  let peak = 0;
  let count = 0;
  // A chunk may end halfway through a sample, whose first byte waits here.
  let pending: number | null = null;
  const take = (sample: number) => {
    peak = Math.max(peak, Math.abs(sample) / 32768);
    if (++count === PEAKS_WINDOW) {
      windows.push(peak);
      peak = 0;
      count = 0;
    }
  };
  for await (const chunk of child.stdout) {
    let i = 0;
    if (pending !== null) {
      take(((chunk[0] << 24) >> 16) | pending);
      pending = null;
      i = 1;
    }
    for (; i + 1 < chunk.length; i += 2) {
      // Little-endian, signed: the high byte is shifted into the sign bit
      // and back.
      take(((chunk[i + 1] << 24) >> 16) | chunk[i]);
    }
    if (i < chunk.length) pending = chunk[i];
  }
  if (count > 0) windows.push(peak);

  const { success } = await child.status;
  if (!success || windows.length === 0) throw new Error("ffmpeg failed");
  return peaksOf(windows);
}

/**
 * Writes the waveform of the audio file `srcPath` as the waveform route's
 * whole response, so the route serves it as a file, with its ETag.
 */
export function audioWaveform(srcPath: string, outPath: string): Promise<void> {
  return writeAtomically(outPath, async (tmpPath) => {
    const body: APIResponse<DumpWaveform> = {
      success: true,
      data: { peaks: await audioPeaks(srcPath) },
    };
    await Deno.writeTextFile(tmpPath, JSON.stringify(body));
  });
}

/**
 * Lets a feed full of cards asking for the same new still at once wait on a
 * single ffmpeg run. It holds the generations under way, by output path.
 */
const inFlight = new Map<string, Promise<void>>();

/**
 * Spares an ffmpeg run per request for an image it can't read: the
 * generations that failed, by output path, with the date of the source they
 * failed on. A new upload gets a new try.
 */
const failed = new Map<string, number>();

/** Returns the date `path` was last written, or null when it doesn't exist. */
async function writtenAt(path: string): Promise<number | null> {
  try {
    return (await Deno.stat(path)).mtime?.getTime() ?? 0;
  } catch {
    return null;
  }
}

/**
 * Makes `outPath` with `generate` unless it exists. Resolves false when it
 * can't be made, because ffmpeg is missing or failed.
 *
 * With `srcPath`, the file it is made from, `outPath` takes that file's date,
 * and is made again when the two differ: an upload replaced in place, as an
 * avatar or a cover is, gets new copies without anyone deleting the old.
 * Without it, the caller deletes `outPath` when its source changes.
 */
export async function ensureGenerated(
  outPath: string,
  generate: () => Promise<void>,
  srcPath?: string,
): Promise<boolean> {
  const made = await writtenAt(outPath);
  let version: number | null = null;
  if (srcPath) {
    version = await writtenAt(srcPath);
    if (version === null) return false;
    if (made === version) return true;
    if (failed.get(outPath) === version) return false;
  } else if (made !== null) {
    return true;
  }
  if (!await ffmpegAvailable()) return false;
  let pending = inFlight.get(outPath);
  if (!pending) {
    pending = generate()
      .then(() =>
        version === null
          ? undefined
          : Deno.utime(outPath, new Date(), new Date(version))
      )
      .catch((err) => {
        if (version !== null) failed.set(outPath, version);
        throw err;
      })
      .finally(() => inFlight.delete(outPath));
    inFlight.set(outPath, pending);
  }
  return await pending.then(() => true, () => false);
}

/**
 * How old a temporary file must be for the sweep to take it: one being
 * written, by a generation under way, is seconds old, and one a crash left
 * is never finished.
 */
const ABANDONED_AFTER_MS = 60 * 60_000;

/** Yields the path of every file under `dir`, and nothing when it is missing. */
async function* filesUnder(dir: string): AsyncGenerator<string> {
  let entries: Deno.DirEntry[];
  try {
    entries = await Array.fromAsync(Deno.readDir(dir));
  } catch {
    return;
  }
  for (const entry of entries) {
    const path = `${dir}/${entry.name}`;
    if (entry.isDirectory) yield* filesUnder(path);
    else if (entry.isFile) yield path;
  }
}

/**
 * Deletes the generated files nothing serves any more, and returns how many:
 * the scaled copies of an upload that is gone, which the services' own
 * removals missed or never made, as for a user's avatar; the previews of
 * image dumps under their former name, `<id>.preview.jpg` or `.png`, which
 * the scaled copies replaced; and the temporary files a generation left
 * unfinished. Nothing else of `THUMBNAILS_DIR` is touched, the custom
 * thumbnails kept there being uploads.
 */
export async function sweepGeneratedMedia(): Promise<number> {
  let removed = 0;
  const remove = (path: string) =>
    Deno.remove(path).then(() => {
      removed++;
    }, () => {});
  const abandoned = async (path: string) => {
    const at = await writtenAt(path);
    return at !== null && Date.now() - at > ABANDONED_AFTER_MS;
  };

  for await (const path of filesUnder(THUMBNAILS_DIR)) {
    if (
      /\.preview\.(jpg|png)$/.test(path) ||
      (path.endsWith(".tmp") && await abandoned(path))
    ) await remove(path);
  }

  for await (const path of filesUnder(SCALED_DIR)) {
    // A copy is named after its upload's path: see scaledImagePath.
    const copy = relative(SCALED_DIR, path).match(
      /^(.+)\.w\d+\.(?:webp|jpg|png)$/,
    );
    const unused = copy
      ? await writtenAt(`${UPLOADS_DIR}/${copy[1]}`) === null
      : await abandoned(path);
    if (unused) await remove(path);
  }
  return removed;
}

/**
 * Runs {@link sweepGeneratedMedia} a minute after the server starts, which
 * clears what a new version no longer uses as soon as it is deployed, then
 * once a day. `main.ts` starts it.
 */
export const generatedMediaSweepJob: Job = {
  name: "media-sweep",
  everyMs: 24 * 60 * 60_000,
  firstAfterMs: 60_000,
  async run() {
    const removed = await sweepGeneratedMedia();
    if (removed > 0) {
      console.log(`[media-sweep] removed ${removed} generated files unused`);
    }
  },
};