v4: an audio file's waveform is made once by the server, and the player seeks on one accessible scrubber — every player downloaded the whole file a second time and decoded all of it in the browser to draw 150 bars, so a long FLAC cost tens of megabytes of memory per page, and the waveform could only be clicked: no drag, no keyboard, no slider for a screen reader. GET /api/dumps/:id/waveform now answers an audio dump's peaks, which lib/ffmpeg.ts decodes with ffmpeg on first request, mono at 8 kHz and streamed, keeps beside the thumbnails as the response itself, and serves with an ETag (a ten-minute MP3 takes about a second, then a few hundred bytes). The peaks are taken by shared/waveform.ts's peaksOf, scaled to the loudest so a quiet recording fills the track, and the dump form's preview of a file not yet uploaded decodes it with the same function, at 8 kHz in an OfflineAudioContext. Scrubber replaces Waveform and SeekBar: a role="slider" that seeks where a pointer lets go, follows a drag (the clock shows where it is held), takes arrows, Home and End, and draws the bars as one path with the played part clipped by a nested svg, so it moves smoothly and a timeupdate redraws two attributes instead of 150 rects; audio without a waveform, and video, get a plain bar. MediaPlayer reads whether it plays from the element's events alone, so a click no longer reports the change twice, its handles for PlayerProvider are stable callbacks instead of refs to refs, the drag no longer reattaches every listener, and the <source> children that src made dead are gone, with the mime prop. The file, thumbnail and waveform routes resolve their dump, token and caching through lib/dump-media.ts, and what is generated from an upload is named and removed in lib/ffmpeg.ts, the custom thumbnail by dump-service's customThumbnailPath, instead of paths spelled out in each. The test setup's play() and pause() fire their events as a browser does. 277 backend tests and 684 frontend tests pass, and deno check, tsc, eslint, doc:lint and i18n:compile --strict are clean; checked in the app with a real FLAC on its dump page and in the global player at 1280 and 390px, seeking by keyboard.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
288
api/lib/ffmpeg.ts
Normal file
288
api/lib/ffmpeg.ts
Normal file
@@ -0,0 +1,288 @@
|
||||
/**
|
||||
* Makes what the server derives from a file dump's upload with ffmpeg: a
|
||||
* video's still and an image's scaled-down copy, which the thumbnails route
|
||||
* serves, and an audio file's waveform, which the waveform route serves. Each
|
||||
* is made on its first request and kept in `THUMBNAILS_DIR` under a name this
|
||||
* module gives, until dump-service replaces or deletes the upload and
|
||||
* {@link removeGeneratedMedia} drops them.
|
||||
*
|
||||
* @module
|
||||
*/
|
||||
|
||||
import { THUMBNAILS_DIR } from "../config.ts";
|
||||
import { type DumpWaveform, peaksOf } from "../../shared/waveform.ts";
|
||||
import type { APIResponse } from "../../shared/api.ts";
|
||||
|
||||
/**
|
||||
* Spares a process per uncached request: whether ffmpeg runs is probed once
|
||||
* per process, and kept here.
|
||||
*/
|
||||
let ffmpegProbe: Promise<boolean> | undefined;
|
||||
|
||||
function ffmpegAvailable(): Promise<boolean> {
|
||||
ffmpegProbe ??= (async () => {
|
||||
try {
|
||||
const { success } = await new Deno.Command("ffmpeg", {
|
||||
args: ["-version"],
|
||||
stderr: "null",
|
||||
stdout: "null",
|
||||
}).output();
|
||||
return success;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
return ffmpegProbe;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists the container formats ffmpeg may read a video as. The upload's type is
|
||||
* only what the uploader claimed, and ffmpeg guesses the format from the bytes,
|
||||
* so a "video" could be an HLS or concat playlist pointing it at local files or
|
||||
* internal URLs. With this list, and `-protocol_whitelist file`, ffmpeg reads
|
||||
* nothing but the file, and only as a video.
|
||||
*/
|
||||
const VIDEO_FORMATS =
|
||||
"mov,mp4,m4a,3gp,3g2,mj2,matroska,webm,avi,flv,mpegts,mpeg,ogg,asf,h264,hevc";
|
||||
/**
|
||||
* Lists the formats ffmpeg may read an image as, for the reason given at
|
||||
* {@link VIDEO_FORMATS}.
|
||||
*/
|
||||
const IMAGE_FORMATS = "jpeg_pipe,png_pipe,webp_pipe,bmp_pipe";
|
||||
/**
|
||||
* Lists the formats ffmpeg may read an audio file as, for the reason given at
|
||||
* {@link VIDEO_FORMATS}; an `.m4a` or a `.webm` is read by its container's.
|
||||
*/
|
||||
const AUDIO_FORMATS =
|
||||
"mp3,ogg,flac,wav,aiff,aac,asf,caf,w64,wv,ape,mov,mp4,m4a,matroska,webm";
|
||||
|
||||
/**
|
||||
* Maps the image types that get a scaled-down preview to the preview's format.
|
||||
* A GIF is served as it is, to keep its animation.
|
||||
*/
|
||||
export const PREVIEWABLE_IMAGES: Record<string, "jpg" | "png"> = {
|
||||
"image/jpeg": "jpg",
|
||||
"image/bmp": "jpg",
|
||||
// May carry transparency, which JPEG would fill with black.
|
||||
"image/png": "png",
|
||||
"image/webp": "png",
|
||||
};
|
||||
|
||||
/**
|
||||
* Caps an image preview's width. It covers the widest feed card on a
|
||||
* high-density screen, at a fraction of a phone photo's bytes.
|
||||
*/
|
||||
const PREVIEW_WIDTH = 960;
|
||||
|
||||
/** Returns where the media generated from dump `dumpId`'s upload are kept. */
|
||||
export function generatedPaths(dumpId: string): {
|
||||
still: string;
|
||||
preview: Record<"jpg" | "png", string>;
|
||||
waveform: string;
|
||||
} {
|
||||
const base = `${THUMBNAILS_DIR}/${dumpId}`;
|
||||
return {
|
||||
still: `${base}.jpg`,
|
||||
preview: { jpg: `${base}.preview.jpg`, png: `${base}.preview.png` },
|
||||
waveform: `${base}.waveform.json`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes what was generated from dump `dumpId`'s upload, once dump-service
|
||||
* has replaced or deleted it. The routes make them again from a new one.
|
||||
*/
|
||||
export async function removeGeneratedMedia(dumpId: string): Promise<void> {
|
||||
const { still, preview, waveform } = generatedPaths(dumpId);
|
||||
await Promise.all(
|
||||
[still, preview.jpg, preview.png, waveform].map((path) =>
|
||||
Deno.remove(path).catch(() => {})
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes `outPath` with `write`, which gets a path beside it to fill, renamed
|
||||
* over `outPath` once done, so a concurrent reader never gets half a file.
|
||||
*/
|
||||
async function writeAtomically(
|
||||
outPath: string,
|
||||
write: (tmpPath: string) => Promise<void>,
|
||||
): Promise<void> {
|
||||
await Deno.mkdir(THUMBNAILS_DIR, { recursive: true });
|
||||
const tmpPath = `${outPath}.${crypto.randomUUID()}.tmp`;
|
||||
try {
|
||||
await write(tmpPath);
|
||||
await Deno.rename(tmpPath, outPath);
|
||||
} catch (err) {
|
||||
await Deno.remove(tmpPath).catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs ffmpeg with `args`, writing to `outPath`.
|
||||
*
|
||||
* @throws {Error} When ffmpeg fails.
|
||||
*/
|
||||
function runFfmpeg(args: string[], outPath: string): Promise<void> {
|
||||
return writeAtomically(outPath, async (tmpPath) => {
|
||||
const { success } = await new Deno.Command("ffmpeg", {
|
||||
args: ["-protocol_whitelist", "file", ...args, "-y", tmpPath],
|
||||
stdout: "null",
|
||||
stderr: "null",
|
||||
}).output();
|
||||
if (!success) throw new Error("ffmpeg failed");
|
||||
});
|
||||
}
|
||||
|
||||
/** Writes a 320-pixel-wide JPEG still of a video, one second in. */
|
||||
export function videoStill(srcPath: string, outPath: string): Promise<void> {
|
||||
return runFfmpeg([
|
||||
"-format_whitelist",
|
||||
VIDEO_FORMATS,
|
||||
"-ss",
|
||||
"00:00:01",
|
||||
"-i",
|
||||
srcPath,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-vf",
|
||||
"scale=320:-1",
|
||||
"-f",
|
||||
"image2",
|
||||
"-c:v",
|
||||
"mjpeg",
|
||||
], outPath);
|
||||
}
|
||||
|
||||
/** Writes a copy of an image at most {@link PREVIEW_WIDTH} wide. */
|
||||
export function imagePreview(srcPath: string, outPath: string): Promise<void> {
|
||||
return runFfmpeg([
|
||||
"-format_whitelist",
|
||||
IMAGE_FORMATS,
|
||||
"-i",
|
||||
srcPath,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-vf",
|
||||
`scale='min(${PREVIEW_WIDTH},iw)':-2`,
|
||||
"-f",
|
||||
"image2",
|
||||
...(outPath.endsWith(".png")
|
||||
? ["-c:v", "png"]
|
||||
: ["-c:v", "mjpeg", "-q:v", "4"]),
|
||||
], outPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* The sample rate the audio is decoded at for its waveform. A bar stands for
|
||||
* a second or more of a song, so 8 kHz keeps every peak worth drawing at a
|
||||
* sixth of the samples.
|
||||
*/
|
||||
const PEAKS_SAMPLE_RATE = 8000;
|
||||
/** How many samples each running peak covers before `peaksOf` buckets them. */
|
||||
const PEAKS_WINDOW = 256;
|
||||
|
||||
/**
|
||||
* Decodes the audio of `srcPath` to its waveform's peaks. ffmpeg streams the
|
||||
* samples, mono at {@link PEAKS_SAMPLE_RATE}, and only the peak of each
|
||||
* {@link PEAKS_WINDOW} is kept, so an hour of audio holds a few hundred
|
||||
* kilobytes at most.
|
||||
*
|
||||
* @throws {Error} When ffmpeg fails or the file holds no audio.
|
||||
*/
|
||||
async function audioPeaks(srcPath: string): Promise<number[]> {
|
||||
const child = new Deno.Command("ffmpeg", {
|
||||
args: [
|
||||
"-protocol_whitelist",
|
||||
"file",
|
||||
"-format_whitelist",
|
||||
AUDIO_FORMATS,
|
||||
"-i",
|
||||
srcPath,
|
||||
"-vn",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
String(PEAKS_SAMPLE_RATE),
|
||||
"-f",
|
||||
"s16le",
|
||||
"-",
|
||||
],
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "null",
|
||||
}).spawn();
|
||||
|
||||
const windows: number[] = [];
|
||||
let peak = 0;
|
||||
let count = 0;
|
||||
// A chunk may end halfway through a sample, whose first byte waits here.
|
||||
let pending: number | null = null;
|
||||
const take = (sample: number) => {
|
||||
peak = Math.max(peak, Math.abs(sample) / 32768);
|
||||
if (++count === PEAKS_WINDOW) {
|
||||
windows.push(peak);
|
||||
peak = 0;
|
||||
count = 0;
|
||||
}
|
||||
};
|
||||
for await (const chunk of child.stdout) {
|
||||
let i = 0;
|
||||
if (pending !== null) {
|
||||
take(((chunk[0] << 24) >> 16) | pending);
|
||||
pending = null;
|
||||
i = 1;
|
||||
}
|
||||
for (; i + 1 < chunk.length; i += 2) {
|
||||
// Little-endian, signed: the high byte is shifted into the sign bit
|
||||
// and back.
|
||||
take(((chunk[i + 1] << 24) >> 16) | chunk[i]);
|
||||
}
|
||||
if (i < chunk.length) pending = chunk[i];
|
||||
}
|
||||
if (count > 0) windows.push(peak);
|
||||
|
||||
const { success } = await child.status;
|
||||
if (!success || windows.length === 0) throw new Error("ffmpeg failed");
|
||||
return peaksOf(windows);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the waveform of the audio file `srcPath` as the waveform route's
|
||||
* whole response, so the route serves it as a file, with its ETag.
|
||||
*/
|
||||
export function audioWaveform(srcPath: string, outPath: string): Promise<void> {
|
||||
return writeAtomically(outPath, async (tmpPath) => {
|
||||
const body: APIResponse<DumpWaveform> = {
|
||||
success: true,
|
||||
data: { peaks: await audioPeaks(srcPath) },
|
||||
};
|
||||
await Deno.writeTextFile(tmpPath, JSON.stringify(body));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Lets a feed full of cards asking for the same new still at once wait on a
|
||||
* single ffmpeg run. It holds the generations under way, by output path.
|
||||
*/
|
||||
const inFlight = new Map<string, Promise<void>>();
|
||||
|
||||
/**
|
||||
* Makes `outPath` with `generate` unless it exists. Resolves false when it
|
||||
* can't be made, because ffmpeg is missing or failed.
|
||||
*/
|
||||
export async function ensureGenerated(
|
||||
outPath: string,
|
||||
generate: () => Promise<void>,
|
||||
): Promise<boolean> {
|
||||
if (await Deno.stat(outPath).then(() => true, () => false)) return true;
|
||||
if (!await ffmpegAvailable()) return false;
|
||||
let pending = inFlight.get(outPath);
|
||||
if (!pending) {
|
||||
pending = generate().finally(() => inFlight.delete(outPath));
|
||||
inFlight.set(outPath, pending);
|
||||
}
|
||||
return await pending.then(() => true, () => false);
|
||||
}
|
||||
Reference in New Issue
Block a user