From fac0e2db2873232c75db55d3cc8390b7f7f790c3 Mon Sep 17 00:00:00 2001 From: khannurien Date: Fri, 9 Oct 2026 13:34:22 +0000 Subject: [PATCH] =?UTF-8?q?v4:=20an=20audio=20file's=20waveform=20is=20mad?= =?UTF-8?q?e=20once=20by=20the=20server,=20and=20the=20player=20seeks=20on?= =?UTF-8?q?=20one=20accessible=20scrubber=20=E2=80=94=20every=20player=20d?= =?UTF-8?q?ownloaded=20the=20whole=20file=20a=20second=20time=20and=20deco?= =?UTF-8?q?ded=20all=20of=20it=20in=20the=20browser=20to=20draw=20150=20ba?= =?UTF-8?q?rs,=20so=20a=20long=20FLAC=20cost=20tens=20of=20megabytes=20of?= =?UTF-8?q?=20memory=20per=20page,=20and=20the=20waveform=20could=20only?= =?UTF-8?q?=20be=20clicked:=20no=20drag,=20no=20keyboard,=20no=20slider=20?= =?UTF-8?q?for=20a=20screen=20reader.=20GET=20/api/dumps/:id/waveform=20no?= =?UTF-8?q?w=20answers=20an=20audio=20dump's=20peaks,=20which=20lib/ffmpeg?= =?UTF-8?q?.ts=20decodes=20with=20ffmpeg=20on=20first=20request,=20mono=20?= =?UTF-8?q?at=208=20kHz=20and=20streamed,=20keeps=20beside=20the=20thumbna?= =?UTF-8?q?ils=20as=20the=20response=20itself,=20and=20serves=20with=20an?= =?UTF-8?q?=20ETag=20(a=20ten-minute=20MP3=20takes=20about=20a=20second,?= =?UTF-8?q?=20then=20a=20few=20hundred=20bytes).=20The=20peaks=20are=20tak?= =?UTF-8?q?en=20by=20shared/waveform.ts's=20peaksOf,=20scaled=20to=20the?= =?UTF-8?q?=20loudest=20so=20a=20quiet=20recording=20fills=20the=20track,?= =?UTF-8?q?=20and=20the=20dump=20form's=20preview=20of=20a=20file=20not=20?= =?UTF-8?q?yet=20uploaded=20decodes=20it=20with=20the=20same=20function,?= =?UTF-8?q?=20at=208=20kHz=20in=20an=20OfflineAudioContext.=20Scrubber=20r?= =?UTF-8?q?eplaces=20Waveform=20and=20SeekBar:=20a=20role=3D"slider"=20tha?= =?UTF-8?q?t=20seeks=20where=20a=20pointer=20lets=20go,=20follows=20a=20dr?= =?UTF-8?q?ag=20(the=20clock=20shows=20where=20it=20is=20held),=20takes=20?= =?UTF-8?q?arrows,=20Home=20and=20End,=20and=20draws=20the=20bars=20as=20o?= =?UTF-8?q?ne=20path=20with=20the=20played=20part=20clipped=20by=20a=20nes?= =?UTF-8?q?ted=20svg,=20so=20it=20moves=20smoothly=20and=20a=20timeupdate?= =?UTF-8?q?=20redraws=20two=20attributes=20instead=20of=20150=20rects;=20a?= =?UTF-8?q?udio=20without=20a=20waveform,=20and=20video,=20get=20a=20plain?= =?UTF-8?q?=20bar.=20MediaPlayer=20reads=20whether=20it=20plays=20from=20t?= =?UTF-8?q?he=20element's=20events=20alone,=20so=20a=20click=20no=20longer?= =?UTF-8?q?=20reports=20the=20change=20twice,=20its=20handles=20for=20Play?= =?UTF-8?q?erProvider=20are=20stable=20callbacks=20instead=20of=20refs=20t?= =?UTF-8?q?o=20refs,=20the=20drag=20no=20longer=20reattaches=20every=20lis?= =?UTF-8?q?tener,=20and=20the=20=20children=20that=20src=20made=20?= =?UTF-8?q?dead=20are=20gone,=20with=20the=20mime=20prop.=20The=20file,=20?= =?UTF-8?q?thumbnail=20and=20waveform=20routes=20resolve=20their=20dump,?= =?UTF-8?q?=20token=20and=20caching=20through=20lib/dump-media.ts,=20and?= =?UTF-8?q?=20what=20is=20generated=20from=20an=20upload=20is=20named=20an?= =?UTF-8?q?d=20removed=20in=20lib/ffmpeg.ts,=20the=20custom=20thumbnail=20?= =?UTF-8?q?by=20dump-service's=20customThumbnailPath,=20instead=20of=20pat?= =?UTF-8?q?hs=20spelled=20out=20in=20each.=20The=20test=20setup's=20play()?= =?UTF-8?q?=20and=20pause()=20fire=20their=20events=20as=20a=20browser=20d?= =?UTF-8?q?oes.=20277=20backend=20tests=20and=20684=20frontend=20tests=20p?= =?UTF-8?q?ass,=20and=20deno=20check,=20tsc,=20eslint,=20doc:lint=20and=20?= =?UTF-8?q?i18n:compile=20--strict=20are=20clean;=20checked=20in=20the=20a?= =?UTF-8?q?pp=20with=20a=20real=20FLAC=20on=20its=20dump=20page=20and=20in?= =?UTF-8?q?=20the=20global=20player=20at=201280=20and=20390px,=20seeking?= =?UTF-8?q?=20by=20keyboard.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- API.md | 5 +- api/lib/dump-media.ts | 46 +++ api/lib/ffmpeg.ts | 288 ++++++++++++++++++ api/main.ts | 2 + api/routes/files.ts | 31 +- api/routes/thumbnails.ts | 199 ++---------- api/routes/waveforms.ts | 48 +++ api/services/dump-service.ts | 39 +-- api/test/routes/waveforms.test.ts | 108 +++++++ locales/en.po | 24 +- locales/fr.po | 24 +- shared/test/waveform.test.ts | 17 ++ shared/waveform.ts | 48 +++ src/App.css | 103 ++++--- src/components/dump/FilePreview.tsx | 29 +- .../dump/create/LocalFilePreview.tsx | 12 +- src/components/player/GlobalPlayer.tsx | 6 +- src/components/player/MediaPlayer.tsx | 153 ++++------ src/components/player/Scrubber.tsx | 151 +++++++++ src/components/player/SeekBar.tsx | 54 ---- src/components/player/Waveform.tsx | 99 ------ src/contexts/player/PlayerContext.ts | 2 + src/hooks/player/usePeaks.ts | 38 +++ src/hooks/player/usePlayFile.ts | 4 + .../components/player/MediaPlayer.test.tsx | 48 +-- src/test/components/player/Scrubber.test.tsx | 99 ++++++ src/test/setup.ts | 19 +- src/test/utils/player/waveform.test.ts | 71 +++-- src/utils/player/waveform.ts | 104 +++---- src/utils/shared/urls.ts | 26 +- 30 files changed, 1217 insertions(+), 680 deletions(-) create mode 100644 api/lib/dump-media.ts create mode 100644 api/lib/ffmpeg.ts create mode 100644 api/routes/waveforms.ts create mode 100644 api/test/routes/waveforms.test.ts create mode 100644 shared/test/waveform.test.ts create mode 100644 shared/waveform.ts create mode 100644 src/components/player/Scrubber.tsx delete mode 100644 src/components/player/SeekBar.tsx delete mode 100644 src/components/player/Waveform.tsx create mode 100644 src/hooks/player/usePeaks.ts create mode 100644 src/test/components/player/Scrubber.test.tsx diff --git a/API.md b/API.md index 2575f1e5..f1e218df 100644 --- a/API.md +++ b/API.md @@ -80,7 +80,8 @@ Every JSON endpoint answers the envelope of `shared/api.ts`: `{ "items", "hasMore" }` and takes `?page=` (from 1) and `?limit=` (20 by default, 100 at most). "Signed in" means a session token as `Authorization: Bearer …`. Media can't send a header, so a private file, -thumbnail, cover or attachment takes the owner's media token as `?token=`. +thumbnail, waveform, cover or attachment takes the owner's media token as +`?token=`. ### me @@ -134,6 +135,8 @@ thumbnail, cover or attachment takes the owner's media token as `?token=`. `dump:moderate`. - `GET`, `PUT`, `DELETE /api/dumps/:id/thumbnail` — the thumbnail; a custom one, set or removed by its poster or `dump:moderate`. +- `GET /api/dumps/:id/waveform` — anyone who may see it. An audio file's + waveform, `{ "peaks" }`, for the player to seek on. - `GET`, `POST /api/dumps/:id/comments` — anyone who may see the dump; posting signed in. Its comments. diff --git a/api/lib/dump-media.ts b/api/lib/dump-media.ts new file mode 100644 index 00000000..91633eaa --- /dev/null +++ b/api/lib/dump-media.ts @@ -0,0 +1,46 @@ +/** + * Resolves the dump whose media a route under `/api/dumps/:dumpId/` serves: + * its file (`routes/files.ts`), its thumbnail (`routes/thumbnails.ts`) or its + * waveform (`routes/waveforms.ts`). Media elements and ``s can't send an + * Authorization header, so these routes take the viewer's media token from + * the query string, and are cached by the file's version. + * + * @module + */ + +import type { Context } from "@oak/oak"; +import { APIException } from "./api-exception.ts"; +import { authenticateMedia } from "./auth.ts"; +import { type Dump, dumpFileVersion } from "../../shared/dump.ts"; +import { UUID_RE } from "../../shared/validation.ts"; +import { getDump } from "../services/dump-service.ts"; + +/** + * Returns dump `dumpId` as the viewer of `ctx`'s `?token=` may see it, and the + * Cache-Control its media get: kept for good under a URL naming the file's + * current version (`?v=`, as `dumpFileVersion` gives), revalidated under any + * other, and never in a shared cache for a private dump. It also keeps the + * token out of the Referer of whatever the response links to. + * + * @throws {APIException} 400 when `dumpId` isn't an id, which names files on + * disk, and 404 when the viewer can't see the dump. + */ +export async function resolveMediaDump( + ctx: Context, + dumpId: string, +): Promise<{ dump: Dump; cacheControl: string }> { + if (!UUID_RE.test(dumpId)) throw new APIException(400, "Invalid dump ID"); + const token = ctx.request.url.searchParams.get("token"); + const viewerId = token ? await authenticateMedia(token) : null; + const dump = getDump(dumpId, viewerId ?? undefined); + ctx.response.headers.set("Referrer-Policy", "no-referrer"); + const scope = dump.isPrivate ? "private" : "public"; + const current = ctx.request.url.searchParams.get("v") === + dumpFileVersion(dump); + return { + dump, + cacheControl: current + ? `${scope}, max-age=31536000, immutable` + : `${scope}, no-cache`, + }; +} diff --git a/api/lib/ffmpeg.ts b/api/lib/ffmpeg.ts new file mode 100644 index 00000000..ae7eb998 --- /dev/null +++ b/api/lib/ffmpeg.ts @@ -0,0 +1,288 @@ +/** + * Makes what the server derives from a file dump's upload with ffmpeg: a + * video's still and an image's scaled-down copy, which the thumbnails route + * serves, and an audio file's waveform, which the waveform route serves. Each + * is made on its first request and kept in `THUMBNAILS_DIR` under a name this + * module gives, until dump-service replaces or deletes the upload and + * {@link removeGeneratedMedia} drops them. + * + * @module + */ + +import { THUMBNAILS_DIR } from "../config.ts"; +import { type DumpWaveform, peaksOf } from "../../shared/waveform.ts"; +import type { APIResponse } from "../../shared/api.ts"; + +/** + * Spares a process per uncached request: whether ffmpeg runs is probed once + * per process, and kept here. + */ +let ffmpegProbe: Promise | undefined; + +function ffmpegAvailable(): Promise { + ffmpegProbe ??= (async () => { + try { + const { success } = await new Deno.Command("ffmpeg", { + args: ["-version"], + stderr: "null", + stdout: "null", + }).output(); + return success; + } catch { + return false; + } + })(); + return ffmpegProbe; +} + +/** + * Lists the container formats ffmpeg may read a video as. The upload's type is + * only what the uploader claimed, and ffmpeg guesses the format from the bytes, + * so a "video" could be an HLS or concat playlist pointing it at local files or + * internal URLs. With this list, and `-protocol_whitelist file`, ffmpeg reads + * nothing but the file, and only as a video. + */ +const VIDEO_FORMATS = + "mov,mp4,m4a,3gp,3g2,mj2,matroska,webm,avi,flv,mpegts,mpeg,ogg,asf,h264,hevc"; +/** + * Lists the formats ffmpeg may read an image as, for the reason given at + * {@link VIDEO_FORMATS}. + */ +const IMAGE_FORMATS = "jpeg_pipe,png_pipe,webp_pipe,bmp_pipe"; +/** + * Lists the formats ffmpeg may read an audio file as, for the reason given at + * {@link VIDEO_FORMATS}; an `.m4a` or a `.webm` is read by its container's. + */ +const AUDIO_FORMATS = + "mp3,ogg,flac,wav,aiff,aac,asf,caf,w64,wv,ape,mov,mp4,m4a,matroska,webm"; + +/** + * Maps the image types that get a scaled-down preview to the preview's format. + * A GIF is served as it is, to keep its animation. + */ +export const PREVIEWABLE_IMAGES: Record = { + "image/jpeg": "jpg", + "image/bmp": "jpg", + // May carry transparency, which JPEG would fill with black. + "image/png": "png", + "image/webp": "png", +}; + +/** + * Caps an image preview's width. It covers the widest feed card on a + * high-density screen, at a fraction of a phone photo's bytes. + */ +const PREVIEW_WIDTH = 960; + +/** Returns where the media generated from dump `dumpId`'s upload are kept. */ +export function generatedPaths(dumpId: string): { + still: string; + preview: Record<"jpg" | "png", string>; + waveform: string; +} { + const base = `${THUMBNAILS_DIR}/${dumpId}`; + return { + still: `${base}.jpg`, + preview: { jpg: `${base}.preview.jpg`, png: `${base}.preview.png` }, + waveform: `${base}.waveform.json`, + }; +} + +/** + * Deletes what was generated from dump `dumpId`'s upload, once dump-service + * has replaced or deleted it. The routes make them again from a new one. + */ +export async function removeGeneratedMedia(dumpId: string): Promise { + const { still, preview, waveform } = generatedPaths(dumpId); + await Promise.all( + [still, preview.jpg, preview.png, waveform].map((path) => + Deno.remove(path).catch(() => {}) + ), + ); +} + +/** + * Writes `outPath` with `write`, which gets a path beside it to fill, renamed + * over `outPath` once done, so a concurrent reader never gets half a file. + */ +async function writeAtomically( + outPath: string, + write: (tmpPath: string) => Promise, +): Promise { + await Deno.mkdir(THUMBNAILS_DIR, { recursive: true }); + const tmpPath = `${outPath}.${crypto.randomUUID()}.tmp`; + try { + await write(tmpPath); + await Deno.rename(tmpPath, outPath); + } catch (err) { + await Deno.remove(tmpPath).catch(() => {}); + throw err; + } +} + +/** + * Runs ffmpeg with `args`, writing to `outPath`. + * + * @throws {Error} When ffmpeg fails. + */ +function runFfmpeg(args: string[], outPath: string): Promise { + return writeAtomically(outPath, async (tmpPath) => { + const { success } = await new Deno.Command("ffmpeg", { + args: ["-protocol_whitelist", "file", ...args, "-y", tmpPath], + stdout: "null", + stderr: "null", + }).output(); + if (!success) throw new Error("ffmpeg failed"); + }); +} + +/** Writes a 320-pixel-wide JPEG still of a video, one second in. */ +export function videoStill(srcPath: string, outPath: string): Promise { + return runFfmpeg([ + "-format_whitelist", + VIDEO_FORMATS, + "-ss", + "00:00:01", + "-i", + srcPath, + "-frames:v", + "1", + "-vf", + "scale=320:-1", + "-f", + "image2", + "-c:v", + "mjpeg", + ], outPath); +} + +/** Writes a copy of an image at most {@link PREVIEW_WIDTH} wide. */ +export function imagePreview(srcPath: string, outPath: string): Promise { + return runFfmpeg([ + "-format_whitelist", + IMAGE_FORMATS, + "-i", + srcPath, + "-frames:v", + "1", + "-vf", + `scale='min(${PREVIEW_WIDTH},iw)':-2`, + "-f", + "image2", + ...(outPath.endsWith(".png") + ? ["-c:v", "png"] + : ["-c:v", "mjpeg", "-q:v", "4"]), + ], outPath); +} + +/** + * The sample rate the audio is decoded at for its waveform. A bar stands for + * a second or more of a song, so 8 kHz keeps every peak worth drawing at a + * sixth of the samples. + */ +const PEAKS_SAMPLE_RATE = 8000; +/** How many samples each running peak covers before `peaksOf` buckets them. */ +const PEAKS_WINDOW = 256; + +/** + * Decodes the audio of `srcPath` to its waveform's peaks. ffmpeg streams the + * samples, mono at {@link PEAKS_SAMPLE_RATE}, and only the peak of each + * {@link PEAKS_WINDOW} is kept, so an hour of audio holds a few hundred + * kilobytes at most. + * + * @throws {Error} When ffmpeg fails or the file holds no audio. + */ +async function audioPeaks(srcPath: string): Promise { + const child = new Deno.Command("ffmpeg", { + args: [ + "-protocol_whitelist", + "file", + "-format_whitelist", + AUDIO_FORMATS, + "-i", + srcPath, + "-vn", + "-ac", + "1", + "-ar", + String(PEAKS_SAMPLE_RATE), + "-f", + "s16le", + "-", + ], + stdin: "null", + stdout: "piped", + stderr: "null", + }).spawn(); + + const windows: number[] = []; + let peak = 0; + let count = 0; + // A chunk may end halfway through a sample, whose first byte waits here. + let pending: number | null = null; + const take = (sample: number) => { + peak = Math.max(peak, Math.abs(sample) / 32768); + if (++count === PEAKS_WINDOW) { + windows.push(peak); + peak = 0; + count = 0; + } + }; + for await (const chunk of child.stdout) { + let i = 0; + if (pending !== null) { + take(((chunk[0] << 24) >> 16) | pending); + pending = null; + i = 1; + } + for (; i + 1 < chunk.length; i += 2) { + // Little-endian, signed: the high byte is shifted into the sign bit + // and back. + take(((chunk[i + 1] << 24) >> 16) | chunk[i]); + } + if (i < chunk.length) pending = chunk[i]; + } + if (count > 0) windows.push(peak); + + const { success } = await child.status; + if (!success || windows.length === 0) throw new Error("ffmpeg failed"); + return peaksOf(windows); +} + +/** + * Writes the waveform of the audio file `srcPath` as the waveform route's + * whole response, so the route serves it as a file, with its ETag. + */ +export function audioWaveform(srcPath: string, outPath: string): Promise { + return writeAtomically(outPath, async (tmpPath) => { + const body: APIResponse = { + success: true, + data: { peaks: await audioPeaks(srcPath) }, + }; + await Deno.writeTextFile(tmpPath, JSON.stringify(body)); + }); +} + +/** + * Lets a feed full of cards asking for the same new still at once wait on a + * single ffmpeg run. It holds the generations under way, by output path. + */ +const inFlight = new Map>(); + +/** + * Makes `outPath` with `generate` unless it exists. Resolves false when it + * can't be made, because ffmpeg is missing or failed. + */ +export async function ensureGenerated( + outPath: string, + generate: () => Promise, +): Promise { + if (await Deno.stat(outPath).then(() => true, () => false)) return true; + if (!await ffmpegAvailable()) return false; + let pending = inFlight.get(outPath); + if (!pending) { + pending = generate().finally(() => inFlight.delete(outPath)); + inFlight.set(outPath, pending); + } + return await pending.then(() => true, () => false); +} diff --git a/api/main.ts b/api/main.ts index 7b39b6e7..7de740ae 100644 --- a/api/main.ts +++ b/api/main.ts @@ -29,6 +29,7 @@ import remoteRouter from "./routes/remote.ts"; import searchRouter from "./routes/search.ts"; import sessionsRouter from "./routes/sessions.ts"; import thumbnailsRouter from "./routes/thumbnails.ts"; +import waveformsRouter from "./routes/waveforms.ts"; import unsubscribeRouter from "./routes/unsubscribe.ts"; import usersRouter from "./routes/users.ts"; import wsRouter from "./routes/ws.ts"; @@ -76,6 +77,7 @@ for ( dumpsRouter, filesRouter, thumbnailsRouter, + waveformsRouter, commentsRouter, playlistsRouter, categoriesRouter, diff --git a/api/routes/files.ts b/api/routes/files.ts index 037f2a7c..fefa3e98 100644 --- a/api/routes/files.ts +++ b/api/routes/files.ts @@ -16,10 +16,9 @@ import { Router } from "@oak/oak"; import { APIException } from "../lib/api-exception.ts"; import type { APIResponse } from "../../shared/api.ts"; -import { type Dump, dumpFileVersion } from "../../shared/dump.ts"; +import type { Dump } from "../../shared/dump.ts"; import { DUMPS_DIR } from "../config.ts"; -import { UUID_RE } from "../../shared/validation.ts"; -import { authenticateMedia } from "../lib/auth.ts"; +import { resolveMediaDump } from "../lib/dump-media.ts"; import { can } from "../lib/permissions.ts"; import { serveFile } from "../lib/serve-file.ts"; import { authMiddleware } from "../middleware/auth.ts"; @@ -66,17 +65,7 @@ function isInlineSafe(mime: string): boolean { router.get("/:dumpId/file", async (ctx) => { const { dumpId } = ctx.params; - - // The id names a file on disk: nothing but an id's characters gets through. - if (!UUID_RE.test(dumpId)) { - throw new APIException(400, "Invalid dump ID"); - } - - // Media elements can't send an Authorization header, so the viewer's media - // token comes in the query string. - const token = ctx.request.url.searchParams.get("token"); - const viewerId = token ? await authenticateMedia(token) : null; - const dump = getDump(dumpId, viewerId ?? undefined); + const { dump, cacheControl } = await resolveMediaDump(ctx, dumpId); if (dump.kind !== "file" || !dump.fileMime || !dump.fileName) { throw new APIException(404, "No file for this dump"); @@ -98,21 +87,9 @@ router.get("/:dumpId/file", async (ctx) => { "sandbox; default-src 'none'; img-src 'self'; media-src 'self'; style-src 'unsafe-inline'", ); } - // The URL may carry a media token; don't pass it on to wherever the file - // links. - ctx.response.headers.set("Referrer-Policy", "no-referrer"); - - // A URL naming the file's current version (dumpFileVersion) always serves the - // same bytes, so it is cached for good; any other is revalidated against the - // ETag. A private dump's bytes must not sit in a shared cache. - const scope = dump.isPrivate ? "private" : "public"; - const current = ctx.request.url.searchParams.get("v") === - dumpFileVersion(dump); const served = await serveFile(ctx, path, { mime: dump.fileMime, - cacheControl: current - ? `${scope}, max-age=31536000, immutable` - : `${scope}, no-cache`, + cacheControl, ranges: true, }); if (!served) { diff --git a/api/routes/thumbnails.ts b/api/routes/thumbnails.ts index ac7870f6..0572ac78 100644 --- a/api/routes/thumbnails.ts +++ b/api/routes/thumbnails.ts @@ -2,8 +2,8 @@ * Serves and changes dumps' thumbnails, a sub-resource of their dump. The * cards and pages that show a file dump build the URL with * `dumpThumbnailUrl`, and the edit page uploads a custom one. Thumbnails made - * from an upload are generated with ffmpeg on first request and kept in - * `THUMBNAILS_DIR`. + * from an upload are generated on first request by lib/ffmpeg.ts, which keeps + * them. * * - `GET /api/dumps/:dumpId/thumbnail?v=&token=` — public; a private dump's * thumbnail needs its poster's media `token`. Serves the dump's custom @@ -22,10 +22,10 @@ import { Router } from "@oak/oak"; import { APIException } from "../lib/api-exception.ts"; import type { APIResponse } from "../../shared/api.ts"; -import { type Dump, dumpFileVersion } from "../../shared/dump.ts"; +import type { Dump } from "../../shared/dump.ts"; import { + customThumbnailPath, getDump, - getDumpThumbnailInfo, removeDumpThumbnail, setDumpThumbnail, } from "../services/dump-service.ts"; @@ -35,9 +35,15 @@ import { serveUploadedFile, validateImageUpload, } from "../lib/upload.ts"; -import { DUMPS_DIR, THUMBNAILS_DIR } from "../config.ts"; -import { UUID_RE } from "../../shared/validation.ts"; -import { authenticateMedia } from "../lib/auth.ts"; +import { DUMPS_DIR } from "../config.ts"; +import { resolveMediaDump } from "../lib/dump-media.ts"; +import { + ensureGenerated, + generatedPaths, + imagePreview, + PREVIEWABLE_IMAGES, + videoStill, +} from "../lib/ffmpeg.ts"; import { can } from "../lib/permissions.ts"; import { type AuthContext, authMiddleware } from "../middleware/auth.ts"; @@ -47,178 +53,20 @@ import { type AuthContext, authMiddleware } from "../middleware/auth.ts"; */ const router: Router = new Router({ prefix: "/api/dumps" }); -/** - * Spares a process per uncached thumbnail request: whether ffmpeg runs is - * probed once per process, and kept here. - */ -let ffmpegProbe: Promise | undefined; - -function ffmpegAvailable(): Promise { - ffmpegProbe ??= (async () => { - try { - const { success } = await new Deno.Command("ffmpeg", { - args: ["-version"], - stderr: "null", - stdout: "null", - }).output(); - return success; - } catch { - return false; - } - })(); - return ffmpegProbe; -} - -/** - * Lists the container formats ffmpeg may read a video as. The upload's type is - * only what the uploader claimed, and ffmpeg guesses the format from the bytes, - * so a "video" could be an HLS or concat playlist pointing it at local files or - * internal URLs. With this list, and `-protocol_whitelist file`, ffmpeg reads - * nothing but the file, and only as a video. - */ -const VIDEO_FORMATS = - "mov,mp4,m4a,3gp,3g2,mj2,matroska,webm,avi,flv,mpegts,mpeg,ogg,asf,h264,hevc"; -/** - * Lists the formats ffmpeg may read an image as, for the reason given at - * {@link VIDEO_FORMATS}. - */ -const IMAGE_FORMATS = "jpeg_pipe,png_pipe,webp_pipe,bmp_pipe"; - -/** - * Maps the image types that get a scaled-down preview to the preview's format. - * A GIF is served as it is, to keep its animation. - */ -const PREVIEWABLE_IMAGES: Record = { - "image/jpeg": "jpg", - "image/bmp": "jpg", - // May carry transparency, which JPEG would fill with black. - "image/png": "png", - "image/webp": "png", -}; - -/** - * Caps an image preview's width. It covers the widest feed card on a - * high-density screen, at a fraction of a phone photo's bytes. - */ -const PREVIEW_WIDTH = 960; - -/** - * Runs ffmpeg with `args`, writing to `outPath`. - * - * @throws {Error} When ffmpeg fails. - */ -async function runFfmpeg(args: string[], outPath: string): Promise { - await Deno.mkdir(THUMBNAILS_DIR, { recursive: true }); - // Written beside and renamed over, so a concurrent reader never gets half. - const tmpPath = `${outPath}.${crypto.randomUUID()}.tmp`; - const { success } = await new Deno.Command("ffmpeg", { - args: ["-protocol_whitelist", "file", ...args, "-y", tmpPath], - stdout: "null", - stderr: "null", - }).output(); - if (!success) { - await Deno.remove(tmpPath).catch(() => {}); - throw new Error("ffmpeg failed"); - } - await Deno.rename(tmpPath, outPath); -} - -/** Writes a 320-pixel-wide JPEG still of a video, one second in. */ -function videoStill(srcPath: string, outPath: string): Promise { - return runFfmpeg([ - "-format_whitelist", - VIDEO_FORMATS, - "-ss", - "00:00:01", - "-i", - srcPath, - "-frames:v", - "1", - "-vf", - "scale=320:-1", - "-f", - "image2", - "-c:v", - "mjpeg", - ], outPath); -} - -/** Writes a copy of an image at most {@link PREVIEW_WIDTH} wide. */ -function imagePreview(srcPath: string, outPath: string): Promise { - return runFfmpeg([ - "-format_whitelist", - IMAGE_FORMATS, - "-i", - srcPath, - "-frames:v", - "1", - "-vf", - `scale='min(${PREVIEW_WIDTH},iw)':-2`, - "-f", - "image2", - ...(outPath.endsWith(".png") - ? ["-c:v", "png"] - : ["-c:v", "mjpeg", "-q:v", "4"]), - ], outPath); -} - -/** - * Lets a feed full of cards asking for the same new still at once wait on a - * single ffmpeg run. It holds the generations under way, by output path. - */ -const inFlight = new Map>(); - -/** - * Makes `outPath` with `generate` unless it exists. Resolves false when it - * can't be made, because ffmpeg is missing or failed. - */ -async function ensureGenerated( - outPath: string, - generate: () => Promise, -): Promise { - if (await Deno.stat(outPath).then(() => true, () => false)) return true; - if (!await ffmpegAvailable()) return false; - let pending = inFlight.get(outPath); - if (!pending) { - pending = generate().finally(() => inFlight.delete(outPath)); - inFlight.set(outPath, pending); - } - return await pending.then(() => true, () => false); -} - router.get("/:dumpId/thumbnail", async (ctx) => { const { dumpId } = ctx.params; + const { dump, cacheControl: generatedCache } = await resolveMediaDump( + ctx, + dumpId, + ); - // The id names files on disk: nothing but an id's characters gets through. - if (!UUID_RE.test(dumpId)) { - throw new APIException(400, "Invalid dump ID"); - } - - // Media elements can't send an Authorization header, so the viewer's media - // token comes in the query string. This privacy check covers the custom - // thumbnail too, since getDumpThumbnailInfo makes none. - const token = ctx.request.url.searchParams.get("token"); - const viewerId = token ? await authenticateMedia(token) : null; - const dump = getDump(dumpId, viewerId ?? undefined); - // The URL may carry a media token, which must not leak through a Referer. - ctx.response.headers.set("Referrer-Policy", "no-referrer"); - // What is generated from the file is cached for good under a URL naming its - // current version (dumpFileVersion), and revalidated under any other. A - // private dump's thumbnail must not sit in a shared cache. - const scope = dump.isPrivate ? "private" : "public"; - const generatedCache = - ctx.request.url.searchParams.get("v") === dumpFileVersion(dump) - ? `${scope}, max-age=31536000, immutable` - : `${scope}, no-cache`; - - const customThumb = getDumpThumbnailInfo(dumpId); - if (customThumb) { + if (dump.thumbnailMime) { // Replacing it doesn't change the file's version, so it is always // revalidated. const served = await serveUploadedFile( ctx, - `${THUMBNAILS_DIR}/${customThumb.id}-custom`, - customThumb.mime, + customThumbnailPath(dump.id), + dump.thumbnailMime, 0, dump.isPrivate, ); @@ -227,9 +75,10 @@ router.get("/:dumpId/thumbnail", async (ctx) => { const mime = dump.kind === "file" ? dump.fileMime ?? "" : ""; const srcPath = `${DUMPS_DIR}/${dump.id}`; + const generated = generatedPaths(dump.id); if (mime.startsWith("video/")) { - const thumbPath = `${THUMBNAILS_DIR}/${dump.id}.jpg`; + const thumbPath = generated.still; if ( !await ensureGenerated(thumbPath, () => videoStill(srcPath, thumbPath)) ) { @@ -249,7 +98,7 @@ router.get("/:dumpId/thumbnail", async (ctx) => { ctx.response.headers.set("X-Content-Type-Options", "nosniff"); const ext = PREVIEWABLE_IMAGES[mime]; if (ext) { - const previewPath = `${THUMBNAILS_DIR}/${dump.id}.preview.${ext}`; + const previewPath = generated.preview[ext]; if ( await ensureGenerated( previewPath, @@ -303,7 +152,7 @@ router.put( // Written before setDumpThumbnail announces it, or clients would fetch it // too early. - await replaceUploadedFile(`${THUMBNAILS_DIR}/${dump.id}-custom`, data); + await replaceUploadedFile(customThumbnailPath(dump.id), data); const responseBody: APIResponse = { success: true, data: setDumpThumbnail(dump.id, mime), diff --git a/api/routes/waveforms.ts b/api/routes/waveforms.ts new file mode 100644 index 00000000..3b58e46d --- /dev/null +++ b/api/routes/waveforms.ts @@ -0,0 +1,48 @@ +/** + * Serves an audio file dump's waveform, a sub-resource of its dump, for the + * player to seek on. The frontend builds the URL with `dumpWaveformUrl`. + * lib/ffmpeg.ts makes it from the upload on first request and keeps it, so a + * listener downloads a few hundred bytes of peaks rather than the file a + * second time to decode it. + * + * - `GET /api/dumps/:dumpId/waveform?v=&token=` — public; a private dump's + * waveform needs its poster's media `token`. Answers a `DumpWaveform`; 404 + * when the dump isn't an audio file or its audio can't be read. + * + * @module + */ + +import { Router } from "@oak/oak"; +import { APIException } from "../lib/api-exception.ts"; +import { DUMPS_DIR } from "../config.ts"; +import { resolveMediaDump } from "../lib/dump-media.ts"; +import { + audioWaveform, + ensureGenerated, + generatedPaths, +} from "../lib/ffmpeg.ts"; +import { serveFile } from "../lib/serve-file.ts"; + +/** + * Mounted by `api/main.ts` behind the error, Open Graph and CORS middleware. + * An `APIException` thrown by a handler becomes the JSON error response there. + */ +const router: Router = new Router({ prefix: "/api/dumps" }); + +router.get("/:dumpId/waveform", async (ctx) => { + const { dump, cacheControl } = await resolveMediaDump(ctx, ctx.params.dumpId); + if (dump.kind !== "file" || !dump.fileMime?.startsWith("audio/")) { + throw new APIException(404, "No waveform for this dump"); + } + + const outPath = generatedPaths(dump.id).waveform; + const srcPath = `${DUMPS_DIR}/${dump.id}`; + if ( + !await ensureGenerated(outPath, () => audioWaveform(srcPath, outPath)) || + !await serveFile(ctx, outPath, { mime: "application/json", cacheControl }) + ) { + throw new APIException(404, "Could not generate waveform"); + } +}); + +export default router; diff --git a/api/services/dump-service.ts b/api/services/dump-service.ts index 55b3ff85..5dc6f5ab 100644 --- a/api/services/dump-service.ts +++ b/api/services/dump-service.ts @@ -40,6 +40,7 @@ import { } from "./notification-service.ts"; import { changeSlug, newSlug, resolveSlug } from "./slug-service.ts"; import { canonicalizeUrl } from "../lib/canonical-url.ts"; +import { removeGeneratedMedia } from "../lib/ffmpeg.ts"; import { DUMP_ALLOWED_MIME_PREFIXES, DUMP_ALLOWED_MIME_TYPES, @@ -236,19 +237,6 @@ export async function createFileDump( return dump; } -/** - * Deletes the thumbnails that `routes/thumbnails.ts` generated from a dump's - * upload, once the upload is replaced or deleted. The route makes them again - * from a new one. - */ -async function removeGeneratedThumbnails(dumpId: string): Promise { - await Promise.all( - [`${dumpId}.jpg`, `${dumpId}.preview.jpg`, `${dumpId}.preview.png`].map( - (name) => Deno.remove(`${THUMBNAILS_DIR}/${name}`).catch(() => {}), - ), - ); -} - /** * Returns dump `dumpId`, without its categories, or throws a 404. It doesn't * check privacy: the routes do, with {@link getDump}. @@ -671,7 +659,7 @@ export async function replaceFileDump( throw err; } if (hadOld) await Deno.remove(backupPath).catch(() => {}); - await removeGeneratedThumbnails(dumpId); + await removeGeneratedMedia(dumpId); if (categoryIds !== undefined) { dump.categoryIds = [...new Set(categoryIds)]; @@ -811,19 +799,12 @@ export async function refreshDumpMetadata(dumpId: string): Promise { } /** - * Returns a dump's id and custom thumbnail type, or undefined without one, for - * the thumbnail route. It doesn't check privacy: the route calls - * {@link getDump} first. - * - * @throws {APIException} 404 when there is no such dump. + * Returns where dump `dumpId`'s custom thumbnail is kept: the thumbnail route + * writes and serves it, and this service deletes it with the thumbnail or the + * dump. */ -export function getDumpThumbnailInfo( - dumpId: string, -): { id: string; mime: string } | undefined { - const dump = fetchDump(dumpId); - return dump.thumbnailMime - ? { id: dump.id, mime: dump.thumbnailMime } - : undefined; +export function customThumbnailPath(dumpId: string): string { + return `${THUMBNAILS_DIR}/${dumpId}-custom`; } /** @@ -853,7 +834,7 @@ export async function removeDumpThumbnail(dumpId: string): Promise { const dump = fetchDump(dumpId); db.prepare(`UPDATE dumps SET custom_thumbnail_mime = NULL WHERE id = ?;`) .run(dump.id); - await Deno.remove(`${THUMBNAILS_DIR}/${dump.id}-custom`).catch(() => {}); + await Deno.remove(customThumbnailPath(dump.id)).catch(() => {}); const updatedDump: Dump = { ...dump, thumbnailMime: undefined }; broadcastDumpChange(dump, updatedDump); return updatedDump; @@ -876,10 +857,10 @@ export async function deleteDump(dumpId: string): Promise { if (dump.kind === "file") { await Deno.remove(`${DUMPS_DIR}/${dump.id}`).catch(() => {}); - await removeGeneratedThumbnails(dump.id); + await removeGeneratedMedia(dump.id); } if (dump.thumbnailMime) { - await Deno.remove(`${THUMBNAILS_DIR}/${dump.id}-custom`).catch(() => {}); + await Deno.remove(customThumbnailPath(dump.id)).catch(() => {}); } broadcastDumpChange(dump, null); diff --git a/api/test/routes/waveforms.test.ts b/api/test/routes/waveforms.test.ts new file mode 100644 index 00000000..548e7776 --- /dev/null +++ b/api/test/routes/waveforms.test.ts @@ -0,0 +1,108 @@ +import { createUser, get, insertDump, test, TEST_DIR } from "../helpers.ts"; +import { assert, assertEquals } from "@std/assert"; +import { WAVEFORM_PEAKS } from "../../../shared/waveform.ts"; + +async function hasFfmpeg(): Promise { + try { + return (await new Deno.Command("ffmpeg", { + args: ["-version"], + stdout: "null", + stderr: "null", + }).output()).success; + } catch { + return false; + } +} + +/** + * Returns a mono 16-bit WAV of a 440 Hz tone, quiet for its first `seconds` + * and four times louder for as long again. + */ +function wav(seconds: number, rate = 8000): Uint8Array { + const n = seconds * rate * 2; + const bytes = new Uint8Array(44 + n * 2); + const view = new DataView(bytes.buffer); + const ascii = (at: number, s: string) => + [...s].forEach((c, i) => view.setUint8(at + i, c.charCodeAt(0))); + ascii(0, "RIFF"); + view.setUint32(4, 36 + n * 2, true); + ascii(8, "WAVEfmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // mono + view.setUint32(24, rate, true); + view.setUint32(28, rate * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + ascii(36, "data"); + view.setUint32(40, n * 2, true); + for (let i = 0; i < n; i++) { + const gain = i < n / 2 ? 4000 : 16000; + view.setInt16( + 44 + i * 2, + gain * Math.sin((2 * Math.PI * 440 * i) / rate), + true, + ); + } + return bytes; +} + +test("GET /api/dumps/:id/waveform: an audio file's peaks, made once, with privacy", async () => { + if (!await hasFfmpeg()) return; + const owner = await createUser(); + const song = insertDump({ + userId: owner.id, + fileName: "s.wav", + fileMime: "audio/wav", + fileData: wav(2), + }); + const res = await get<{ peaks: number[] }>(`/api/dumps/${song.id}/waveform`); + assertEquals(res.status, 200); + assertEquals(res.headers.get("content-type"), "application/json"); + assertEquals(res.headers.get("cache-control"), "public, no-cache"); + const { peaks } = res.data; + assertEquals(peaks.length, WAVEFORM_PEAKS); + // The loud half reaches the top; the quiet half a quarter of it. + assertEquals(Math.max(...peaks), 1); + assert(Math.abs(peaks[10] - 0.25) < 0.02, `quiet peak ${peaks[10]}`); + assert(peaks[140] > 0.98, `loud peak ${peaks[140]}`); + assert( + [...Deno.readDirSync(`${TEST_DIR}/uploads/thumbnails`)].some((e) => + e.name === `${song.id}.waveform.json` + ), + ); + // The kept file is served again, with its ETag. + const etag = res.headers.get("etag")!; + const again = await get(`/api/dumps/${song.id}/waveform`, { + headers: { "If-None-Match": etag }, + }); + assertEquals(again.status, 304); + + const secret = insertDump({ + userId: owner.id, + fileName: "p.wav", + fileMime: "audio/wav", + fileData: wav(1), + isPrivate: true, + }); + assertEquals((await get(`/api/dumps/${secret.id}/waveform`)).status, 404); + assertEquals( + (await get(`/api/dumps/${secret.id}/waveform?token=${owner.mediaToken}`)) + .status, + 200, + ); +}); + +test("GET /api/dumps/:id/waveform: 404 for anything but readable audio", async () => { + const owner = await createUser(); + const text = insertDump({ userId: owner.id, fileName: "t.txt" }); + assertEquals((await get(`/api/dumps/${text.id}/waveform`)).status, 404); + const noise = insertDump({ + userId: owner.id, + fileName: "n.mp3", + fileMime: "audio/mpeg", + fileData: new TextEncoder().encode("not audio at all"), + }); + assertEquals((await get(`/api/dumps/${noise.id}/waveform`)).status, 404); + assertEquals((await get(`/api/dumps/nope/waveform`)).status, 400); +}); diff --git a/locales/en.po b/locales/en.po index 9d276137..cd6828b7 100644 --- a/locales/en.po +++ b/locales/en.po @@ -669,7 +669,7 @@ msgid "Done" msgstr "Done" #. placeholder {0}: dump.fileName -#: src/components/dump/FilePreview.tsx:215 +#: src/components/dump/FilePreview.tsx:226 msgid "Download {0}" msgstr "Download {0}" @@ -911,7 +911,7 @@ msgstr "From people you follow" msgid "From playlists" msgstr "From playlists" -#: src/components/player/MediaPlayer.tsx:373 +#: src/components/player/MediaPlayer.tsx:326 msgid "Fullscreen" msgstr "Fullscreen" @@ -1127,7 +1127,7 @@ msgstr "Moderator" msgid "digest.monthly" msgstr "Monthly recap" -#: src/components/player/MediaPlayer.tsx:346 +#: src/components/player/MediaPlayer.tsx:299 msgid "Mute" msgstr "Mute" @@ -1335,10 +1335,10 @@ msgstr "Passwords do not match" msgid "Paste a link…" msgstr "Paste a link…" -#: src/components/dump/FilePreview.tsx:65 -#: src/components/dump/FilePreview.tsx:177 +#: src/components/dump/FilePreview.tsx:70 +#: src/components/dump/FilePreview.tsx:184 #: src/components/dump/RichContentCard.tsx:103 -#: src/components/player/MediaPlayer.tsx:330 +#: src/components/player/MediaPlayer.tsx:276 msgid "Pause" msgstr "Pause" @@ -1346,11 +1346,11 @@ msgstr "Pause" msgid "Picked up where you left off." msgstr "Picked up where you left off." -#: src/components/dump/FilePreview.tsx:65 -#: src/components/dump/FilePreview.tsx:177 +#: src/components/dump/FilePreview.tsx:70 +#: src/components/dump/FilePreview.tsx:184 #: src/components/dump/RichContentCard.tsx:103 #: src/components/dump/RichContentCard.tsx:150 -#: src/components/player/MediaPlayer.tsx:330 +#: src/components/player/MediaPlayer.tsx:276 msgid "Play" msgstr "Play" @@ -1571,7 +1571,7 @@ msgstr "Searching…" msgid "email.digest.communityPage" msgstr "See it all on the community page" -#: src/components/player/SeekBar.tsx:50 +#: src/components/player/Scrubber.tsx:101 msgid "Seek" msgstr "Seek" @@ -1809,7 +1809,7 @@ msgstr "Unfollow {targetUsername}" msgid "Unfollow playlist" msgstr "Unfollow playlist" -#: src/components/player/MediaPlayer.tsx:346 +#: src/components/player/MediaPlayer.tsx:299 msgid "Unmute" msgstr "Unmute" @@ -1902,7 +1902,7 @@ msgstr "View all →" msgid "View dump →" msgstr "View dump →" -#: src/components/player/MediaPlayer.tsx:363 +#: src/components/player/MediaPlayer.tsx:316 msgid "Volume" msgstr "Volume" diff --git a/locales/fr.po b/locales/fr.po index a86dddd0..dfebb83d 100644 --- a/locales/fr.po +++ b/locales/fr.po @@ -669,7 +669,7 @@ msgid "Done" msgstr "Terminé" #. placeholder {0}: dump.fileName -#: src/components/dump/FilePreview.tsx:215 +#: src/components/dump/FilePreview.tsx:226 msgid "Download {0}" msgstr "Télécharger {0}" @@ -911,7 +911,7 @@ msgstr "De ceux que vous suivez" msgid "From playlists" msgstr "De collections" -#: src/components/player/MediaPlayer.tsx:373 +#: src/components/player/MediaPlayer.tsx:326 msgid "Fullscreen" msgstr "Plein écran" @@ -1127,7 +1127,7 @@ msgstr "Modérateur" msgid "digest.monthly" msgstr "Récap mensuel" -#: src/components/player/MediaPlayer.tsx:346 +#: src/components/player/MediaPlayer.tsx:299 msgid "Mute" msgstr "Couper le son" @@ -1335,10 +1335,10 @@ msgstr "Les mots de passe ne correspondent pas" msgid "Paste a link…" msgstr "Collez un lien…" -#: src/components/dump/FilePreview.tsx:65 -#: src/components/dump/FilePreview.tsx:177 +#: src/components/dump/FilePreview.tsx:70 +#: src/components/dump/FilePreview.tsx:184 #: src/components/dump/RichContentCard.tsx:103 -#: src/components/player/MediaPlayer.tsx:330 +#: src/components/player/MediaPlayer.tsx:276 msgid "Pause" msgstr "Pause" @@ -1346,11 +1346,11 @@ msgstr "Pause" msgid "Picked up where you left off." msgstr "Reprise de votre brouillon." -#: src/components/dump/FilePreview.tsx:65 -#: src/components/dump/FilePreview.tsx:177 +#: src/components/dump/FilePreview.tsx:70 +#: src/components/dump/FilePreview.tsx:184 #: src/components/dump/RichContentCard.tsx:103 #: src/components/dump/RichContentCard.tsx:150 -#: src/components/player/MediaPlayer.tsx:330 +#: src/components/player/MediaPlayer.tsx:276 msgid "Play" msgstr "Lecture" @@ -1571,7 +1571,7 @@ msgstr "Recherche…" msgid "email.digest.communityPage" msgstr "Tout voir sur la page de la communauté" -#: src/components/player/SeekBar.tsx:50 +#: src/components/player/Scrubber.tsx:101 msgid "Seek" msgstr "Position" @@ -1809,7 +1809,7 @@ msgstr "Ne plus suivre {targetUsername}" msgid "Unfollow playlist" msgstr "Ne plus suivre la collection" -#: src/components/player/MediaPlayer.tsx:346 +#: src/components/player/MediaPlayer.tsx:299 msgid "Unmute" msgstr "Rétablir le son" @@ -1902,7 +1902,7 @@ msgstr "Tout voir →" msgid "View dump →" msgstr "Voir la reco →" -#: src/components/player/MediaPlayer.tsx:363 +#: src/components/player/MediaPlayer.tsx:316 msgid "Volume" msgstr "Volume" diff --git a/shared/test/waveform.test.ts b/shared/test/waveform.test.ts new file mode 100644 index 00000000..708f5944 --- /dev/null +++ b/shared/test/waveform.test.ts @@ -0,0 +1,17 @@ +import { assertEquals } from "@std/assert"; +import { peaksOf } from "../waveform.ts"; + +Deno.test("peaksOf: each slice's loudest sample, scaled to the loudest", () => { + assertEquals(peaksOf([0.1, -0.5, 0.2, 0.25, -0.1, 0.05], 3), [1, 0.5, 0.2]); +}); + +Deno.test("peaksOf: every sample counts, however they divide", () => { + assertEquals(peaksOf([0, 0, 0, 0, 0, 0, 0.8], 3), [0, 0, 1]); + // Fewer samples than peaks: neighbours share one. + assertEquals(peaksOf([0.5, 1], 4), [0.5, 0.5, 1, 1]); +}); + +Deno.test("peaksOf: silence is flat, not divided by zero", () => { + assertEquals(peaksOf([0, 0, 0], 2), [0, 0]); + assertEquals(peaksOf([], 2), [0, 0]); +}); diff --git a/shared/waveform.ts b/shared/waveform.ts new file mode 100644 index 00000000..38deb286 --- /dev/null +++ b/shared/waveform.ts @@ -0,0 +1,48 @@ +/** + * Defines an audio file's waveform, the bars the player seeks on: how many + * peaks it has, and how they are taken from the samples. The server makes an + * uploaded file's from its decoded audio (`api/lib/ffmpeg.ts`), and the + * dump form a local file's in the browser (`src/utils/player/waveform.ts`), + * both with {@link peaksOf}, so the two draw alike. + * + * @module + */ + +/** How many peaks, and so bars, a waveform has. */ +export const WAVEFORM_PEAKS = 150; + +/** + * Returned by `GET /api/dumps/:dumpId/waveform` for an audio file dump, to the + * player's waveform. + */ +export interface DumpWaveform { + /** {@link WAVEFORM_PEAKS} heights from 0 to 1, as {@link peaksOf} gives. */ + peaks: number[]; +} + +/** + * Returns `n` peaks of `samples`, each the loudest of its slice, scaled so the + * loudest of all is 1 and rounded to three places: a quiet recording fills the + * waveform as a loud one does. `samples` may be signed, as audio is, or + * already peaks of smaller slices. Silence gives zeros. + */ +export function peaksOf( + samples: ArrayLike, + n: number = WAVEFORM_PEAKS, +): number[] { + const peaks: number[] = []; + for (let i = 0; i < n; i++) { + // Slices cover every sample; with fewer samples than peaks, some share. + const start = Math.floor((i * samples.length) / n); + const end = Math.max(start + 1, Math.floor(((i + 1) * samples.length) / n)); + let peak = 0; + for (let j = start; j < Math.min(end, samples.length); j++) { + peak = Math.max(peak, Math.abs(samples[j])); + } + peaks.push(peak); + } + const loudest = Math.max(...peaks); + return peaks.map((p) => + loudest > 0 ? Math.round((p / loudest) * 1000) / 1000 : 0 + ); +} diff --git a/src/App.css b/src/App.css index 938e0bcd..dea84a4e 100644 --- a/src/App.css +++ b/src/App.css @@ -1063,10 +1063,9 @@ border-radius: 0 0 var(--radius-xl) var(--radius-xl); } -/* When active, brighten only the *un-played* bars. Excluding played bars keeps - this rule from out-specifying `.waveform-bar--played` (which would paint every - bar the played colour and make the waveform look stuck at 100%). */ -.audio-file-preview--active .waveform-bar:not(.waveform-bar--played) { +/* When active, brighten the bars still to play: the outer path only, since the + played copy sits in the nested svg. */ +.audio-file-preview--active .scrubber-peaks > path { fill: color-mix(in srgb, var(--color-accent) 45%, var(--color-border) 55%); } @@ -1213,75 +1212,75 @@ ); } -/* ── Waveform ── */ -.waveform-svg { +/* ── Scrubber: the track MediaPlayer seeks on, a waveform or a plain bar ── */ +.scrubber { + position: relative; flex: 1; - min-width: 0; + min-width: 48px; height: 48px; + display: flex; + align-items: center; cursor: pointer; - display: block; + /* A drag on a phone scrubs rather than scrolls the page. */ + touch-action: none; + border-radius: var(--radius-xs); } -.waveform-bar { +.scrubber:focus-visible { + outline: 2px solid var(--color-accent); + outline-offset: 2px; +} + +/* A video's controls overlay the picture, so its track stays thin. */ +.scrubber--slim { + height: 1rem; +} + +.scrubber-peaks { + display: block; + width: 100%; + height: 100%; +} + +.scrubber-peaks path { fill: color-mix(in srgb, var(--color-accent) 25%, var(--color-border) 75%); } -.waveform-bar--played { +/* The same bars, clipped to the playhead by this nested svg's width. */ +.scrubber-peaks-played { + overflow: hidden; +} + +.scrubber-peaks-played path { fill: var(--color-accent); } -.waveform-skeleton { - flex: 1; - min-width: 0; - height: 48px; +.scrubber-bar { + position: relative; + width: 100%; + height: 6px; border-radius: var(--radius-xs); + overflow: hidden; background: color-mix( in srgb, var(--color-accent) 12%, var(--color-border) 88% ); - position: relative; - overflow: hidden; - cursor: pointer; } -.waveform-skeleton-fill { +.scrubber--slim .scrubber-bar { + height: 4px; +} + +.video-player-controls .scrubber-bar { + background: rgba(255, 255, 255, 0.25); +} + +.scrubber-fill { position: absolute; inset: 0 auto 0 0; background: var(--color-accent); - opacity: 0.45; - transition: width var(--dur-fast) linear; -} - -/* Audio whose peaks can't be decoded (cross-origin streams). Boxed to the - waveform's height so the player doesn't resize between item kinds, with the - bar itself centred inside. */ -.audio-player-track--stream { - height: 48px; - background: none; - display: flex; - align-items: center; -} -.audio-player-track--stream::before { - content: ""; - position: absolute; - inset: 50% 0 auto 0; - transform: translateY(-50%); - height: 6px; - border-radius: var(--radius-xs); - background: color-mix( - in srgb, - var(--color-accent) 12%, - var(--color-border) 88% - ); -} -.audio-player-track--stream .audio-player-fill { - top: 50%; - bottom: auto; - transform: translateY(-50%); - height: 6px; - border-radius: var(--radius-xs); - z-index: var(--z-raised); + pointer-events: none; } .audio-player-track--volume { @@ -1341,7 +1340,7 @@ } /* A narrow bar keeps the mute button but drops the volume slider, which took - as much room as the waveform (a phone's volume is in its buttons anyway). */ + as much room as the scrubber (a phone's volume is in its buttons anyway). */ @container (max-width: 420px) { .audio-player-track--volume { display: none; diff --git a/src/components/dump/FilePreview.tsx b/src/components/dump/FilePreview.tsx index e34f5047..5ad8ceae 100644 --- a/src/components/dump/FilePreview.tsx +++ b/src/components/dump/FilePreview.tsx @@ -8,7 +8,11 @@ import { useContext } from "react"; import type { Dump } from "../../../shared/dump.ts"; import { formatBytes } from "../../utils/shared/format.ts"; -import { dumpFileUrl, dumpThumbnailUrl } from "../../utils/shared/urls.ts"; +import { + dumpFileUrl, + dumpThumbnailUrl, + dumpWaveformUrl, +} from "../../utils/shared/urls.ts"; import { useAuth } from "../../hooks/auth/useAuth.ts"; import { fileArtworkUrl, @@ -16,7 +20,8 @@ import { usePlayFile, } from "../../hooks/player/usePlayFile.ts"; import { MediaPlayer } from "../player/MediaPlayer.tsx"; -import { Waveform } from "../player/Waveform.tsx"; +import { Scrubber } from "../player/Scrubber.tsx"; +import { usePeaks } from "../../hooks/player/usePeaks.ts"; import { Thumbnail } from "../shared/Thumbnail.tsx"; import { PlayerContext, @@ -84,7 +89,9 @@ function FilePlayButton({ dump }: { dump: Dump }) { function AudioFilePreview({ dump }: { dump: Dump }) { const { seekTo } = useContext(PlayerContext); const { currentTime, duration } = useContext(PlayerTimeContext); - const { fileUrl, active, toggle } = usePlayFile(dump); + const { mediaToken: token } = useAuth(); + const { active, toggle } = usePlayFile(dump); + const peaks = usePeaks({ peaksUrl: dumpWaveformUrl(dump, token) }); return (
- active ? seekTo(time) : toggle()} + peaks={peaks} />
); @@ -190,14 +197,18 @@ export function FilePreview( ); } - return ( - - ); + return ; } if (mime.startsWith("audio/")) { if (useGlobal) return ; - return ; + return ( + + ); } if (mime === "application/pdf") { diff --git a/src/components/dump/create/LocalFilePreview.tsx b/src/components/dump/create/LocalFilePreview.tsx index 3bbada00..a37917af 100644 --- a/src/components/dump/create/LocalFilePreview.tsx +++ b/src/components/dump/create/LocalFilePreview.tsx @@ -38,10 +38,18 @@ export function LocalFilePreview( return {file.name}; } if (mime.startsWith("video/")) { - return ; + return ; } if (mime.startsWith("audio/")) { - return ; + // Not uploaded yet, so its waveform is decoded here. + return ( + + ); } return null; } diff --git a/src/components/player/GlobalPlayer.tsx b/src/components/player/GlobalPlayer.tsx index 3753bd21..a47eb7d3 100644 --- a/src/components/player/GlobalPlayer.tsx +++ b/src/components/player/GlobalPlayer.tsx @@ -289,7 +289,9 @@ export function GlobalPlayer(): React.JSX.Element | null { key={current.fileUrl} src={current.fileUrl} kind={current.mimeType.startsWith("video/") ? "video" : "audio"} - mime={current.mimeType} + waveform={current.waveformUrl + ? { peaksUrl: current.waveformUrl } + : undefined} autoplay={autoplay} startTime={startTime} onPlayStateChange={onPlayStateChange} @@ -308,8 +310,6 @@ export function GlobalPlayer(): React.JSX.Element | null { key={current.streamUrl} src={current.streamUrl} kind="audio" - mime="audio/mpeg" - trackStyle="progress" autoplay={autoplay} startTime={startTime} onPlayStateChange={onPlayStateChange} diff --git a/src/components/player/MediaPlayer.tsx b/src/components/player/MediaPlayer.tsx index a2c76323..c592a2d3 100644 --- a/src/components/player/MediaPlayer.tsx +++ b/src/components/player/MediaPlayer.tsx @@ -6,7 +6,7 @@ * @module */ -import { useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { fmt } from "../../utils/player/duration.ts"; import { t } from "@lingui/core/macro"; import { @@ -15,8 +15,9 @@ import { IconPlay, IconVolume, } from "./PlayerIcons.tsx"; -import { SeekBar } from "./SeekBar.tsx"; -import { Waveform } from "./Waveform.tsx"; +import { Scrubber } from "./Scrubber.tsx"; +import { usePeaks } from "../../hooks/player/usePeaks.ts"; +import type { PeaksSource } from "../../utils/player/waveform.ts"; import { rememberVolume, storedVolume } from "../../utils/player/volume.ts"; /** How long a playing video's controls stay after the pointer stops, in ms. */ @@ -30,8 +31,6 @@ const HIDE_DELAY = 2500; export interface MediaPlayerProps { src: string; kind: "audio" | "video"; - /** The `` type, when known. */ - mime?: string; /** Shown before a video plays. */ poster?: string; autoplay?: boolean; @@ -47,10 +46,10 @@ export interface MediaPlayerProps { */ onError?: () => void; /** - * "waveform" (the default for audio) decodes peaks from the source; - * "progress" draws a seek bar and never fetches the media a second time. + * Where an audio file's waveform comes from. Without it, as for a stream + * whose bytes another site serves, audio seeks on a plain bar. */ - trackStyle?: "waveform" | "progress"; + waveform?: PeaksSource; /** Receives its seek handle while mounted, for PlayerContext's `seekTo`. */ seekRef?: { current: ((t: number) => void) | null }; /** Receives its play/pause handle while mounted, for `togglePlay`. */ @@ -58,17 +57,17 @@ export interface MediaPlayerProps { } /** - * Renders a media element with the site's own controls. Audio gets a waveform - * to seek on, unless `trackStyle` asks for a seek bar; video gets a seek bar, a - * fullscreen button, and controls that hide while it plays. The volume carries - * over to the next MediaPlayer, which matters since GlobalPlayer mounts a new - * one for each track. + * Renders a media element with the site's own controls. Audio seeks on its + * waveform when it has one; video gets a slim bar, a fullscreen button, and + * controls that hide while it plays. Whether it plays is read from the + * element's own events, so a media key or the OS's controls show too. The + * volume carries over to the next MediaPlayer, which matters since + * GlobalPlayer mounts a new one for each track. */ export function MediaPlayer( { src, kind, - mime, poster, autoplay, startTime, @@ -76,7 +75,7 @@ export function MediaPlayer( onTimeUpdate, onEnded, onError, - trackStyle = "waveform", + waveform, seekRef, toggleRef, }: MediaPlayerProps, @@ -85,11 +84,13 @@ export function MediaPlayer( const [playing, setPlaying] = useState(false); const [current, setCurrent] = useState(0); const [duration, setDuration] = useState(0); - const [dragging, setDragging] = useState(false); + // Where the scrubber is held, which the clock shows until it lets go. + const [scrubTime, setScrubTime] = useState(null); const [volume, setVolume] = useState(() => storedVolume().volume); const [muted, setMuted] = useState(() => storedVolume().muted); const [controlsVisible, setControlsVisible] = useState(true); const hideTimer = useRef | null>(null); + const peaks = usePeaks(kind === "audio" ? waveform : undefined); // The media listeners call the latest callbacks through these refs, without // being attached again. They are written after render, as the linter wants; @@ -106,40 +107,18 @@ export function MediaPlayer( onErrorRef.current = onError; }); - // seekRef and toggleRef get wrappers that call these, so the handles never go - // null while the functions are replaced. - const seekToFnRef = useRef((_t: number) => {}); - const toggleFnRef = useRef(() => {}); - - // Everything it uses is stable, so it is set once. - useEffect(() => { - seekToFnRef.current = (time: number) => { - setCurrent(time); - mediaRef.current!.currentTime = time; - }; + // Both read the element alone, so they never change, and the handles given + // to PlayerProvider stay valid while this is mounted. + const seekTo = useCallback((time: number) => { + setCurrent(time); + mediaRef.current!.currentTime = time; + }, []); + // The play and pause events update the state, whoever asked. + const toggle = useCallback(() => { + const a = mediaRef.current!; + if (a.paused) a.play().catch(() => {}); + else a.pause(); }, []); - - // It reads `playing`, so it is replaced when that changes. - useEffect(() => { - toggleFnRef.current = () => { - const a = mediaRef.current!; - if (playing) { - a.pause(); - setPlaying(false); - onPlayStateChangeRef.current?.(false); - } else { - a.play() - .then(() => { - setPlaying(true); - onPlayStateChangeRef.current?.(true); - }) - .catch(() => {}); - } - }; - }, [playing]); - - const seekTo = (time: number) => seekToFnRef.current(time); - const toggle = () => toggleFnRef.current(); // The element can't seek before its metadata loads. useEffect(() => { @@ -165,18 +144,6 @@ export function MediaPlayer( // eslint-disable-next-line react-hooks/exhaustive-deps }, []); - // The browser may refuse to play without a user gesture, and the player then - // just shows paused. - useEffect(() => { - if (!autoplay) return; - mediaRef.current?.play() - .then(() => { - setPlaying(true); - onPlayStateChangeRef.current?.(true); - }) - .catch(() => {}); - }, [autoplay]); - // Cutting the callbacks on unmount keeps a late timeupdate or ended event of // this element from reaching PlayerProvider once the next item is playing. useEffect(() => { @@ -192,22 +159,19 @@ export function MediaPlayer( // The refs are PlayerProvider's, which never change, so this runs once. useEffect(() => { - if (seekRef) seekRef.current = (t) => seekToFnRef.current(t); - if (toggleRef) toggleRef.current = () => toggleFnRef.current(); + if (seekRef) seekRef.current = seekTo; + if (toggleRef) toggleRef.current = toggle; return () => { if (seekRef) seekRef.current = null; if (toggleRef) toggleRef.current = null; }; - }, [seekRef, toggleRef]); + }, [seekRef, toggleRef, seekTo, toggle]); useEffect(() => { const a = mediaRef.current!; const onTime = () => { - // While the seek bar is dragged, its thumb follows the pointer. - if (!dragging) { - setCurrent(a.currentTime); - onTimeUpdateRef.current?.(a.currentTime, a.duration); - } + setCurrent(a.currentTime); + onTimeUpdateRef.current?.(a.currentTime, a.duration); }; const onDuration = () => { setDuration(a.duration); @@ -222,9 +186,6 @@ export function MediaPlayer( setPlaying(false); onErrorRef.current?.(); }; - // The element can also be started or stopped from outside the page — a - // media key, a headset, the OS's media controls — so follow it rather - // than only what our own button did. const onPlayEvent = () => { setPlaying(true); onPlayStateChangeRef.current?.(true); @@ -247,7 +208,13 @@ export function MediaPlayer( a.removeEventListener("ended", onEndedEvent); a.removeEventListener("error", onErrorEvent); }; - }, [dragging]); + }, []); + + // After the listeners, which the play event must find. The browser may + // refuse to play without a user gesture, and the player then shows paused. + useEffect(() => { + if (autoplay) mediaRef.current?.play().catch(() => {}); + }, [autoplay]); useEffect(() => { if (kind !== "video") return; @@ -300,27 +267,6 @@ export function MediaPlayer( (mediaRef.current as HTMLVideoElement).requestFullscreen?.(); }; - // A waveform needs the raw bytes, which cross-origin streams won't hand over. - // Those get an honest progress bar rather than a permanent loading skeleton. - const track = kind === "audio" && trackStyle === "waveform" - ? ( - - ) - : ( - - ); - const controls = ( <> - {fmt(current)} + {fmt(scrubTime ?? current)} - {track} + {fmt(duration)} @@ -394,9 +347,7 @@ export function MediaPlayer( preload="metadata" className="video-player-video" onClick={toggle} - > - {mime && } - + />
{controls}
@@ -410,9 +361,7 @@ export function MediaPlayer( ref={mediaRef as React.RefObject} src={src} preload="metadata" - > - {mime && } - + /> {controls} ); diff --git a/src/components/player/Scrubber.tsx b/src/components/player/Scrubber.tsx new file mode 100644 index 00000000..4331cfdc --- /dev/null +++ b/src/components/player/Scrubber.tsx @@ -0,0 +1,151 @@ +/** + * Renders the track MediaPlayer seeks on: an audio file's waveform, or a plain + * bar for a stream, a video, or audio whose waveform isn't in yet. The dump + * page's audio preview shows one too, beside its cover. + * + * @module + */ + +import { useMemo, useState } from "react"; +import { t } from "@lingui/core/macro"; +import { fmt } from "../../utils/player/duration.ts"; + +/** A bar's width, in viewBox units. */ +const BAR_W = 3; +/** The gap between bars, in viewBox units. */ +const BAR_GAP = 1; +/** The waveform's height, in viewBox units. */ +const WAVEFORM_H = 48; +/** How far an arrow key seeks, in seconds. */ +const KEY_STEP = 5; + +/** Passed to {@link Scrubber} by MediaPlayer and FilePreview. Times are in seconds. */ +export interface ScrubberProps { + current: number; + duration: number; + /** Called once a click, a drag or a key lands somewhere. */ + onSeek: (time: number) => void; + /** + * Called with the time under the pointer while it holds the scrubber, and + * with null once it lets go, so MediaPlayer's clock can follow the drag. + */ + onScrub?: (time: number | null) => void; + /** The waveform's heights from 0 to 1; without them, a plain bar. */ + peaks?: number[] | null; + /** Draws the thin bar of a video's controls instead of the audio track. */ + slim?: boolean; +} + +/** + * Returns the waveform's bars as one path, each `BAR_W` wide and centred on + * the middle line, so a redraw for the playhead touches two attributes rather + * than a hundred and fifty elements. + */ +function barsPath(peaks: number[]): string { + return peaks.map((p, i) => { + const h = Math.max(p * WAVEFORM_H, 2); + const x = i * (BAR_W + BAR_GAP); + return `M${x} ${(WAVEFORM_H - h) / 2}h${BAR_W}v${h}h-${BAR_W}z`; + }).join(""); +} + +/** + * Renders a slider over the media's timeline. A pointer seeks where it lets + * go, showing where it is while held; arrows seek {@link KEY_STEP} seconds, + * Home and End to either end. The played part of the waveform is the same + * path clipped to the playhead by a nested ``, so it moves smoothly + * within a bar. + */ +export function Scrubber( + { current, duration, onSeek, onScrub, peaks, slim = false }: ScrubberProps, +): React.JSX.Element { + // The fraction under the pointer while it holds the scrubber. + const [held, setHeld] = useState(null); + const path = useMemo(() => peaks && barsPath(peaks), [peaks]); + + const shown = held !== null ? held * duration : current; + const progress = duration > 0 + ? Math.min(1, Math.max(0, shown / duration)) + : 0; + + const fractionAt = (e: React.PointerEvent) => { + const rect = e.currentTarget.getBoundingClientRect(); + if (rect.width <= 0) return 0; + return Math.min(1, Math.max(0, (e.clientX - rect.left) / rect.width)); + }; + const hold = (fraction: number | null) => { + setHeld(fraction); + onScrub?.(fraction === null ? null : fraction * duration); + }; + + const onKeyDown = (e: React.KeyboardEvent) => { + const to = { + ArrowLeft: current - KEY_STEP, + ArrowDown: current - KEY_STEP, + ArrowRight: current + KEY_STEP, + ArrowUp: current + KEY_STEP, + Home: 0, + End: duration, + }[e.key]; + if (to === undefined || !(duration > 0)) return; + e.preventDefault(); + onSeek(Math.min(duration, Math.max(0, to))); + }; + + const width = peaks ? peaks.length * (BAR_W + BAR_GAP) - BAR_GAP : 0; + + return ( +
{ + if (e.button !== 0) return; + // Keeps the pointer's moves coming here when it leaves the track. + e.currentTarget.setPointerCapture?.(e.pointerId); + hold(fractionAt(e)); + }} + onPointerMove={(e) => held !== null && hold(fractionAt(e))} + onPointerUp={(e) => { + if (held === null) return; + const fraction = fractionAt(e); + hold(null); + onSeek(fraction * duration); + }} + onPointerCancel={() => held !== null && hold(null)} + onKeyDown={onKeyDown} + > + {path + ? ( + + ) + : ( +
+
+
+ )} +
+ ); +} diff --git a/src/components/player/SeekBar.tsx b/src/components/player/SeekBar.tsx deleted file mode 100644 index 94b87d43..00000000 --- a/src/components/player/SeekBar.tsx +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Renders the slim seek bar MediaPlayer shows for video and for streams. - * - * @module - */ - -import { t } from "@lingui/core/macro"; - -/** Passed to {@link SeekBar} by MediaPlayer. Times are in seconds. */ -export interface SeekBarProps { - current: number; - duration: number; - onSeek: (t: number) => void; - /** - * Called as the thumb is grabbed and let go, so that MediaPlayer stops moving - * it with the playhead in between. - */ - onDragChange: (dragging: boolean) => void; - className?: string; -} - -/** - * Renders a progress bar with a range input over it to seek. MediaPlayer uses - * it for video, and for streams, whose bytes can't be read cross-origin to - * draw a waveform. - */ -export function SeekBar( - { current, duration, onSeek, onDragChange, className }: SeekBarProps, -): React.JSX.Element { - const progress = duration > 0 ? current / duration : 0; - return ( -
-
- - onDragChange(true)} - onPointerUp={() => onDragChange(false)} - onPointerCancel={() => onDragChange(false)} - onChange={(e) => onSeek(Number(e.target.value))} - aria-label={t`Seek`} - /> -
- ); -} diff --git a/src/components/player/Waveform.tsx b/src/components/player/Waveform.tsx deleted file mode 100644 index 47753803..00000000 --- a/src/components/player/Waveform.tsx +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Renders an audio file's waveform, which MediaPlayer shows as its seek bar and - * an audio file's dump page beside its cover. - * - * @module - */ - -import { useEffect, useState } from "react"; -import { - BAR_GAP, - BAR_W, - extractPeaks, - NUM_BARS, - VIEWBOX_W, - WAVEFORM_H, -} from "../../utils/player/waveform.ts"; - -/** - * Passed to {@link Waveform} by MediaPlayer and FilePreview. Times are in - * seconds. - */ -export interface WaveformProps { - /** Fetched again, apart from the media element, to decode its peaks. */ - src: string; - current: number; - duration: number; - onSeek: (t: number) => void; -} - -/** - * Renders the peaks of `src` as bars, the played ones highlighted, and seeks - * where clicked. Until the peaks are decoded, or if they can't be, it shows a - * plain progress bar that seeks all the same. - */ -export function Waveform( - { src, current, duration, onSeek }: WaveformProps, -): React.JSX.Element { - const [peaks, setPeaks] = useState(null); - - useEffect(() => { - let cancelled = false; - extractPeaks(src, NUM_BARS) - .then((p) => { - if (!cancelled) setPeaks(p); - }) - .catch(() => {}); - return () => { - cancelled = true; - }; - }, [src]); - - const progress = duration > 0 ? current / duration : 0; - - const handleClick = (e: React.MouseEvent) => { - const rect = e.currentTarget.getBoundingClientRect(); - const ratio = Math.max( - 0, - Math.min(1, (e.clientX - rect.left) / rect.width), - ); - onSeek(ratio * duration); - }; - - if (!peaks) { - return ( -
-
-
- ); - } - - return ( - - {Array.from(peaks).map((p, i) => { - const barH = Math.max(p * WAVEFORM_H, 2); - const x = i * (BAR_W + BAR_GAP); - const y = (WAVEFORM_H - barH) / 2; - const played = i / NUM_BARS <= progress; - return ( - - ); - })} - - ); -} diff --git a/src/contexts/player/PlayerContext.ts b/src/contexts/player/PlayerContext.ts index 35f3fa38..8eba7ae1 100644 --- a/src/contexts/player/PlayerContext.ts +++ b/src/contexts/player/PlayerContext.ts @@ -45,6 +45,8 @@ export type PlayerItem = fileUrl: string; /** Decides between the audio and the video player. */ mimeType: string; + /** An audio file's waveform (`dumpWaveformUrl`), for the player's track. */ + waveformUrl?: string; }) /** * A track streamed from another site, today a Bandcamp mp3. Its URL is diff --git a/src/hooks/player/usePeaks.ts b/src/hooks/player/usePeaks.ts new file mode 100644 index 00000000..c871ffe4 --- /dev/null +++ b/src/hooks/player/usePeaks.ts @@ -0,0 +1,38 @@ +/** + * Gives MediaPlayer the waveform it draws on its `Scrubber`. + * + * @module + */ + +import { useEffect, useState } from "react"; +import { + loadPeaks, + peaksKey, + type PeaksSource, +} from "../../utils/player/waveform.ts"; + +/** + * Returns the peaks of `source` once loaded, and null until then, without a + * source, or when it can't be loaded: the scrubber then draws a plain bar. + */ +export function usePeaks(source: PeaksSource | undefined): number[] | null { + const key = source && peaksKey(source); + const [loaded, setLoaded] = useState<{ key: string; peaks: number[] }>(); + + useEffect(() => { + if (!source) return; + let cancelled = false; + loadPeaks(source).then( + (peaks) => !cancelled && setLoaded({ key: peaksKey(source), peaks }), + () => {}, + ); + return () => { + cancelled = true; + }; + // The source is a new object on each render; its key says when it changed. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [key]); + + // A load for another source may still be in, until this one's arrives. + return loaded && loaded.key === key ? loaded.peaks : null; +} diff --git a/src/hooks/player/usePlayFile.ts b/src/hooks/player/usePlayFile.ts index 32f9b953..e9fbff26 100644 --- a/src/hooks/player/usePlayFile.ts +++ b/src/hooks/player/usePlayFile.ts @@ -17,6 +17,7 @@ import { dumpFileUrl, dumpThumbnailUrl, dumpUrl, + dumpWaveformUrl, } from "../../utils/shared/urls.ts"; /** Checks whether `dump` is an uploaded file the global player can play. */ @@ -76,6 +77,9 @@ export function usePlayFile(dump: Dump): UsePlayFileResult { kind: "file", fileUrl, mimeType: dump.fileMime ?? "", + waveformUrl: dump.fileMime?.startsWith("audio/") + ? dumpWaveformUrl(dump, token) + : undefined, title: dump.title, dumpHref: dumpUrl(dump), artworkUrl: fileArtworkUrl(dump, token), diff --git a/src/test/components/player/MediaPlayer.test.tsx b/src/test/components/player/MediaPlayer.test.tsx index cc70db7e..f60c4994 100644 --- a/src/test/components/player/MediaPlayer.test.tsx +++ b/src/test/components/player/MediaPlayer.test.tsx @@ -26,7 +26,6 @@ describe("MediaPlayer", () => { , ); @@ -44,7 +43,6 @@ describe("MediaPlayer", () => { , @@ -61,7 +59,6 @@ describe("MediaPlayer", () => { , ); @@ -75,7 +72,6 @@ describe("MediaPlayer", () => { , ); @@ -95,7 +91,6 @@ describe("MediaPlayer", () => { , @@ -105,28 +100,46 @@ describe("MediaPlayer", () => { expect(screen.getByText("1:05")).toBeInTheDocument(); expect(screen.getByText("3:20")).toBeInTheDocument(); expect(onTimeUpdate).toHaveBeenLastCalledWith(65, 200); - fireEvent.change(screen.getByRole("slider", { name: "Seek" }), { - target: { value: "120" }, + // Five seconds on with an arrow, as the scrubber's keyboard does. + fireEvent.keyDown(screen.getByRole("slider", { name: "Seek" }), { + key: "ArrowRight", }); - expect(el.currentTime).toBe(120); + expect(el.currentTime).toBe(70); fireEvent(el, new Event("ended")); expect(onEnded).toHaveBeenCalled(); }); - test("while the seek bar is held, playback doesn't move the thumb — by touch too", () => { + test("while the scrubber is held, the clock follows it, not playback", () => { const { container } = renderWithContexts( - , + , ); const el = media(container); setTime(el, 10, 100); const seek = screen.getByRole("slider", { name: "Seek" }); - fireEvent.pointerDown(seek, { pointerType: "touch" }); - fireEvent.change(seek, { target: { value: "50" } }); + seek.getBoundingClientRect = () => new DOMRect(0, 0, 200, 48); + fireEvent.pointerDown(seek, { button: 0, clientX: 100 }); setTime(el, 11, 100); expect(screen.getByText("0:50")).toBeInTheDocument(); - fireEvent.pointerUp(seek, { pointerType: "touch" }); - setTime(el, 51, 100); - expect(screen.getByText("0:51")).toBeInTheDocument(); + expect(el.currentTime).toBe(11); + fireEvent.pointerUp(seek, { clientX: 150 }); + expect(el.currentTime).toBe(75); + expect(screen.getByText("1:15")).toBeInTheDocument(); + }); + + test("an audio file draws its waveform once loaded", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + Response.json({ success: true, data: { peaks: [0.5, 1, 0.25] } }), + ); + const { container } = renderWithContexts( + , + ); + await waitFor(() => + expect(container.querySelector(".scrubber-peaks")).not.toBeNull() + ); }); test("an error is reported for re-resolving", () => { @@ -135,7 +148,6 @@ describe("MediaPlayer", () => { , ); @@ -145,7 +157,7 @@ describe("MediaPlayer", () => { test("volume and mute are remembered", async () => { const { container } = renderWithContexts( - , + , ); fireEvent.change(screen.getByRole("slider", { name: "Volume" }), { target: { value: "0.4" }, @@ -172,7 +184,6 @@ describe("MediaPlayer", () => { , @@ -190,7 +201,6 @@ describe("MediaPlayer", () => { , ); diff --git a/src/test/components/player/Scrubber.test.tsx b/src/test/components/player/Scrubber.test.tsx new file mode 100644 index 00000000..0556259e --- /dev/null +++ b/src/test/components/player/Scrubber.test.tsx @@ -0,0 +1,99 @@ +import { describe, expect, test, vi } from "vitest"; +import { fireEvent, screen } from "@testing-library/react"; +import { Scrubber } from "../../../components/player/Scrubber.tsx"; +import { renderWithContexts } from "../../render.tsx"; + +function slider() { + const el = screen.getByRole("slider", { name: "Seek" }); + el.getBoundingClientRect = () => new DOMRect(10, 0, 200, 48); + return el; +} + +describe("Scrubber", () => { + test("states its position for assistive tech", () => { + renderWithContexts( + {}} />, + ); + const el = slider(); + expect(el).toHaveAttribute("aria-valuenow", "65"); + expect(el).toHaveAttribute("aria-valuemax", "200"); + expect(el).toHaveAttribute("aria-valuetext", "1:05 / 3:20"); + }); + + test("keys seek by five seconds, Home and End to the ends, within bounds", () => { + const onSeek = vi.fn(); + renderWithContexts( + , + ); + const el = slider(); + fireEvent.keyDown(el, { key: "ArrowRight" }); + expect(onSeek).toHaveBeenLastCalledWith(8); + fireEvent.keyDown(el, { key: "ArrowLeft" }); + expect(onSeek).toHaveBeenLastCalledWith(0); + fireEvent.keyDown(el, { key: "End" }); + expect(onSeek).toHaveBeenLastCalledWith(200); + fireEvent.keyDown(el, { key: "Tab" }); + expect(onSeek).toHaveBeenCalledTimes(3); + }); + + test("a drag shows where it is held and seeks where it lets go", () => { + const onSeek = vi.fn(); + const onScrub = vi.fn(); + const { container } = renderWithContexts( + , + ); + const el = slider(); + const fill = () => + container.querySelector(".scrubber-fill")!.style.width; + fireEvent.pointerDown(el, { button: 0, clientX: 60 }); + expect(onScrub).toHaveBeenLastCalledWith(25); + expect(fill()).toBe("25%"); + // Past the end clamps. + fireEvent.pointerMove(el, { clientX: 400 }); + expect(fill()).toBe("100%"); + expect(onSeek).not.toHaveBeenCalled(); + fireEvent.pointerUp(el, { clientX: 110 }); + expect(onSeek).toHaveBeenCalledWith(50); + expect(onScrub).toHaveBeenLastCalledWith(null); + // A move with nothing held does nothing. + fireEvent.pointerMove(el, { clientX: 30 }); + expect(onScrub).toHaveBeenCalledTimes(3); + }); + + test("a cancelled drag seeks nowhere", () => { + const onSeek = vi.fn(); + renderWithContexts( + , + ); + const el = slider(); + fireEvent.pointerDown(el, { button: 0, clientX: 60 }); + fireEvent.pointerCancel(el); + expect(onSeek).not.toHaveBeenCalled(); + }); + + test("peaks draw as bars, the played part clipped to the playhead", () => { + const { container } = renderWithContexts( + {}} + peaks={[0.5, 1, 0.25, 0]} + />, + ); + const svg = container.querySelector(".scrubber-peaks")!; + // Four bars, 3 wide with 1 between: 15 units. + expect(svg.getAttribute("viewBox")).toBe("0 0 15 48"); + expect(svg.querySelector(":scope > path")!.getAttribute("d")).toBe( + "M0 12h3v24h-3zM4 0h3v48h-3zM8 18h3v12h-3zM12 23h3v2h-3z", + ); + expect( + container.querySelector(".scrubber-peaks-played")!.getAttribute("width"), + ).toBe("3.75"); + expect(container.querySelector(".scrubber-bar")).toBeNull(); + }); +}); diff --git a/src/test/setup.ts b/src/test/setup.ts index 07fa03f2..741345e5 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -150,10 +150,25 @@ class FakeAnimation extends EventTarget implements Animation { Element.prototype.animate ??= () => new FakeAnimation(); Element.prototype.getAnimations ??= () => []; globalThis.scrollTo = () => {}; -HTMLMediaElement.prototype.play = function () { +// jsdom plays nothing. These keep `paused` and fire `play` and `pause` as a +// browser does, which the players' state follows. +const playingMedia = new WeakSet(); +Object.defineProperty(HTMLMediaElement.prototype, "paused", { + configurable: true, + get(this: HTMLMediaElement) { + return !playingMedia.has(this); + }, +}); +HTMLMediaElement.prototype.play = function (this: HTMLMediaElement) { + if (!playingMedia.has(this)) { + playingMedia.add(this); + this.dispatchEvent(new Event("play")); + } return Promise.resolve(); }; -HTMLMediaElement.prototype.pause = function () {}; +HTMLMediaElement.prototype.pause = function (this: HTMLMediaElement) { + if (playingMedia.delete(this)) this.dispatchEvent(new Event("pause")); +}; HTMLMediaElement.prototype.load = function () {}; // The emoji picker probes emoji support on a canvas; jsdom has none. Object.defineProperty(HTMLCanvasElement.prototype, "getContext", { diff --git a/src/test/utils/player/waveform.test.ts b/src/test/utils/player/waveform.test.ts index 24825348..8453784a 100644 --- a/src/test/utils/player/waveform.test.ts +++ b/src/test/utils/player/waveform.test.ts @@ -1,40 +1,51 @@ import { describe, expect, test, vi } from "vitest"; -import { extractPeaks } from "../../../utils/player/waveform.ts"; +import { loadPeaks } from "../../../utils/player/waveform.ts"; import { calls, fail, route } from "../../fetch.ts"; -describe("extractPeaks", () => { - test("decodes once per URL and takes each block's peak", async () => { - const data = new Float32Array([0.1, -0.5, 0.2, 0.9, -0.3, 0.4]); - const decode = vi.fn(() => Promise.resolve({ getChannelData: () => data })); - const close = vi.fn(); +describe("loadPeaks", () => { + test("an uploaded file's come from the server, once per URL", async () => { + route( + "GET /api/dumps/:id/waveform", + () => Response.json({ success: true, data: { peaks: [0.5, 1] } }), + ); + const source = { + peaksUrl: "http://localhost:8000/api/dumps/w1/waveform", + }; + expect(await loadPeaks(source)).toEqual([0.5, 1]); + await loadPeaks(source); + expect(calls("GET /api/dumps/:id/waveform")).toHaveLength(1); + }); + + test("a failed load is tried again", async () => { + route("GET /api/dumps/:id/waveform", () => fail("gone", 404)); + const source = { + peaksUrl: "http://localhost:8000/api/dumps/w2/waveform", + }; + await expect(loadPeaks(source)).rejects.toThrow("gone"); + await expect(loadPeaks(source)).rejects.toThrow("gone"); + expect(calls("GET /api/dumps/:id/waveform")).toHaveLength(2); + }); + + test("a local file is decoded here, at 8 kHz, into the shared peaks", async () => { + const data = new Float32Array([0.1, -0.5, 0.2, 0.9, -0.3, 0.45]); + const created: unknown[][] = []; vi.stubGlobal( - "AudioContext", + "OfflineAudioContext", class { - decodeAudioData = decode; - close = close; + constructor(...args: unknown[]) { + created.push(args); + } + decodeAudioData = () => Promise.resolve({ getChannelData: () => data }); }, ); - route( - "GET /api/dumps/:id/file", - () => new Response(new Uint8Array([1, 2])), + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(new Uint8Array([1, 2])), ); - - const peaks = await extractPeaks( - "http://localhost:8000/api/dumps/w1/file", - 3, - ); - expect([...peaks].map((p) => +p.toFixed(2))).toEqual([0.5, 0.9, 0.4]); - expect(close).toHaveBeenCalled(); - - await extractPeaks("http://localhost:8000/api/dumps/w1/file", 3); - expect(decode).toHaveBeenCalledTimes(1); - expect(calls("GET /api/dumps/:id/file")).toHaveLength(1); - }); - - test("a failed fetch throws", async () => { - route("GET /api/dumps/:id/file", () => fail("gone", 404)); - await expect(extractPeaks("http://localhost:8000/api/dumps/w2/file", 3)) - .rejects - .toThrow(/404/); + const peaks = await loadPeaks({ localFile: "blob:http://x/local-1" }); + // Fewer samples than peaks: each sample's slice is shared, scaled to 0.9. + expect(peaks).toHaveLength(150); + expect(peaks[0]).toBe(0.111); + expect(Math.max(...peaks)).toBe(1); + expect(created).toEqual([[1, 1, 8000]]); }); }); diff --git a/src/utils/player/waveform.ts b/src/utils/player/waveform.ts index da2f39f4..c9a2be0c 100644 --- a/src/utils/player/waveform.ts +++ b/src/utils/player/waveform.ts @@ -1,64 +1,66 @@ /** - * Decodes an audio file into the peaks of its waveform, and sets the waveform's - * geometry. MediaPlayer's `Waveform` and the dump page's audio preview - * (`FilePreview`) both draw with it. + * Loads an audio file's waveform for `Scrubber`, by way of `usePeaks`: an + * uploaded file's from the server, which made it once with ffmpeg, and a + * file not yet uploaded (the dump form's preview) by decoding it here. * * @module */ -/** How many bars the waveform draws, one per peak. */ -export const NUM_BARS = 150; -/** A bar's width, in viewBox units. */ -export const BAR_W = 3; -/** The gap between bars, in viewBox units. */ -export const BAR_GAP = 1; -/** The waveform's height, in viewBox units. */ -export const WAVEFORM_H = 48; -/** The waveform's width, in viewBox units. */ -export const VIEWBOX_W = NUM_BARS * (BAR_W + BAR_GAP); +import { type DumpWaveform, peaksOf } from "../../../shared/waveform.ts"; +import { expectOk } from "../shared/api.ts"; /** - * Kept at module level, so that remounts and StrictMode's double effects don't - * fetch and decode a file again. + * Where a waveform's peaks come from: the server's for an uploaded file + * (`dumpWaveformUrl`), or the bytes of a local file, by its `blob:` URL. */ -const peaksCache = new Map(); +export type PeaksSource = { peaksUrl: string } | { localFile: string }; /** - * Returns `n` peaks of the audio at `url`, each the loudest sample of its slice - * of the first channel. Each URL is fetched and decoded once per page load. - * - * @throws {Error} When the fetch or the decoding fails. + * Kept at module level, by URL, so remounts, StrictMode's double effects and + * the several players of one file share a load. A failed one is dropped, to be + * tried again by the next player. */ -export async function extractPeaks( - url: string, - n: number, -): Promise { - if (peaksCache.has(url)) return peaksCache.get(url)!; +const loads = new Map>(); - // No credentials: the server's CORS doesn't allow them, and the file URL - // carries the viewer's media token already.