v3: a performance pass over both halves — indexed queries, streamed files, cached previews, and feeds that stop re-rendering on every tick — a review turned up whole-file reads, unindexed scans and render storms; each fix comes with a test. Database: migration 0016 adds the indexes the hot queries lacked — dumps by created_at, (user_id, created_at) and slug, votes by (user_id, created_at), playlists by slug, users by lower(username) for @mentions, and the child side of every cascading foreign key (comment replies, a user's comments, follows both ways, chat messages, invites, reset tokens); the feeds, dump and playlist pages, upvotes and follower fan-out all plan as index searches now, with no temp-table sort. SQLite runs in WAL with synchronous=NORMAL and a busy timeout, and a transaction() helper wraps the multi-row writes (a dump and its categories, category sets, backlinks, attachment links, playlist reorders, follower notifications) so they commit once instead of once a row. Lists no longer run a COUNT(*) per page: they fetch limit + 1 rows and hasMore comes from the extra one; total stays only on search, the one place it's shown. Files: /api/files read the whole upload (up to 50 MB) for every range request a player makes while seeking; api/lib/serve-file.ts streams the requested bytes from disk, with an ETag and 304s, and serves avatars, attachments, covers and thumbnails too. File and thumbnail URLs carry the file's version (size and last change), so the current one is cached for a year and anything else revalidates — a same-size replacement used to stay stale, and a video's still was cached immutable forever under a URL that never changed. Feeds showed image dumps at full size: /api/thumbnails now makes a 960px preview with ffmpeg (GIFs and unreadable images go out as they are), one run per output however many cards ask at once, and ffmpeg is probed once per process instead of spawned per request. Uploads stream to disk; replacing a file writes beside it and renames over, a hard link keeping the old one for rollback instead of holding both in memory; generated stills and previews go when the file is replaced or the dump deleted, and a custom thumbnail with its dump. Outbound: fetchWithTimeout's deadline now covers the body — a server that answered at once and then dripped could hold a create open indefinitely — and pages are read to 4 MB at most; metadata is cached five minutes so the create form's preview and the post that follows share one fetch (refresh-metadata still goes to the page); proxied images sit in a 32 MB LRU. Broadcasts serialize once rather than per socket, and index.html is kept in memory until the file changes. Frontend: currentTime lived in PlayerContext, so every card re-rendered — and re-parsed its Markdown — four times a second while anything played; the playhead has its own PlayerTimeContext now. The socket context is split three ways (feed, chat via useWSChat, presence via useWSPresence), so typing, presence and chat messages leave the feeds alone. DumpCard, JournalCard and Markdown are memoized; card images use the server preview and load lazily; the feed cache is written when the page is left rather than after every pause in scrolling; the theme context value is memoized.
All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 49s

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
2026-09-27 16:35:51 +00:00
parent 364c498a24
commit de5abeceb2
74 changed files with 1795 additions and 792 deletions

View File

@@ -1,59 +1,128 @@
import { Router } from "@oak/oak";
import { APIErrorCode, APIException } from "../model/interfaces.ts";
import { getDump, getDumpThumbnailInfo } from "../services/dump-service.ts";
import {
dumpFileVersion,
getDump,
getDumpThumbnailInfo,
} from "../services/dump-service.ts";
import { serveFile } from "../lib/serve-file.ts";
import { serveUploadedFile } from "../lib/upload.ts";
import { DUMPS_DIR, THUMBNAILS_DIR } from "../config.ts";
import { authenticateMedia } from "../lib/auth.ts";
const router = new Router({ prefix: "/api/thumbnails" });
async function ffmpegAvailable(): Promise<boolean> {
try {
const cmd = new Deno.Command("ffmpeg", {
args: ["-version"],
stderr: "null",
stdout: "null",
});
const { success } = await cmd.output();
return success;
} catch {
return false;
}
// Probed once per process: spawning `ffmpeg -version` on every uncached
// thumbnail request cost a process per card.
let ffmpegProbe: Promise<boolean> | undefined;
function ffmpegAvailable(): Promise<boolean> {
ffmpegProbe ??= (async () => {
try {
const { success } = await new Deno.Command("ffmpeg", {
args: ["-version"],
stderr: "null",
stdout: "null",
}).output();
return success;
} catch {
return false;
}
})();
return ffmpegProbe;
}
async function generateThumbnail(
srcPath: string,
outPath: string,
): Promise<void> {
// The upload's type is only what the uploader claimed, and ffmpeg probes the
// real format from the bytes — a "video" could be an HLS or concat playlist
// pointing ffmpeg at local files or internal URLs. Read nothing but the file
// itself, and only as the kind of container it's meant to be.
const VIDEO_FORMATS =
"mov,mp4,m4a,3gp,3g2,mj2,matroska,webm,avi,flv,mpegts,mpeg,ogg,asf,h264,hevc";
const IMAGE_FORMATS = "jpeg_pipe,png_pipe,webp_pipe,bmp_pipe";
/** Images the feed shows through a scaled-down preview. GIFs keep their
* animation (served as they are); anything else has no preview to make. */
const PREVIEWABLE_IMAGES: Record<string, "jpg" | "png"> = {
"image/jpeg": "jpg",
"image/bmp": "jpg",
// May carry transparency, which JPEG would fill with black.
"image/png": "png",
"image/webp": "png",
};
/** Width of an image preview: comfortably covers the widest feed card on a
* high-density screen, a fraction of a phone photo's bytes. */
const PREVIEW_WIDTH = 960;
async function runFfmpeg(args: string[], outPath: string): Promise<void> {
await Deno.mkdir(THUMBNAILS_DIR, { recursive: true });
const cmd = new Deno.Command("ffmpeg", {
args: [
// The upload's type is only what the uploader claimed, and ffmpeg probes
// the real format from the bytes — a "video" could be an HLS or concat
// playlist pointing ffmpeg at local files or internal URLs. Read nothing
// but the file itself, and only as an actual video container.
"-protocol_whitelist",
"file",
"-format_whitelist",
"mov,mp4,m4a,3gp,3g2,mj2,matroska,webm,avi,flv,mpegts,mpeg,ogg,asf,h264,hevc",
"-ss",
"00:00:01",
"-i",
srcPath,
"-frames:v",
"1",
"-vf",
"scale=320:-1",
"-f",
"image2",
"-y",
outPath,
],
// Written beside and renamed over, so a concurrent reader never gets half.
const tmpPath = `${outPath}.${crypto.randomUUID()}.tmp`;
const { success } = await new Deno.Command("ffmpeg", {
args: ["-protocol_whitelist", "file", ...args, "-y", tmpPath],
stdout: "null",
stderr: "null",
});
const { success } = await cmd.output();
if (!success) throw new Error("ffmpeg failed");
}).output();
if (!success) {
await Deno.remove(tmpPath).catch(() => {});
throw new Error("ffmpeg failed");
}
await Deno.rename(tmpPath, outPath);
}
function videoStill(srcPath: string, outPath: string): Promise<void> {
return runFfmpeg([
"-format_whitelist",
VIDEO_FORMATS,
"-ss",
"00:00:01",
"-i",
srcPath,
"-frames:v",
"1",
"-vf",
"scale=320:-1",
"-f",
"image2",
"-c:v",
"mjpeg",
], outPath);
}
function imagePreview(srcPath: string, outPath: string): Promise<void> {
return runFfmpeg([
"-format_whitelist",
IMAGE_FORMATS,
"-i",
srcPath,
"-frames:v",
"1",
"-vf",
`scale='min(${PREVIEW_WIDTH},iw)':-2`,
"-f",
"image2",
...(outPath.endsWith(".png")
? ["-c:v", "png"]
: ["-c:v", "mjpeg", "-q:v", "4"]),
], outPath);
}
// One ffmpeg run per output: a feed full of cards asking for the same new
// still at once waits on a single generation.
const inFlight = new Map<string, Promise<void>>();
async function ensureGenerated(
outPath: string,
generate: () => Promise<void>,
): Promise<boolean> {
if (await Deno.stat(outPath).then(() => true, () => false)) return true;
if (!await ffmpegAvailable()) return false;
let pending = inFlight.get(outPath);
if (!pending) {
pending = generate().finally(() => inFlight.delete(outPath));
inFlight.set(outPath, pending);
}
return await pending.then(() => true, () => false);
}
router.get("/:dumpId", async (ctx) => {
@@ -70,67 +139,80 @@ router.get("/:dumpId", async (ctx) => {
const dump = getDump(dumpId, viewerId ?? undefined);
// The URL may carry a media token; keep it out of anyone else's logs.
ctx.response.headers.set("Referrer-Policy", "no-referrer");
// A private dump's thumbnail must not sit in a shared cache.
const cacheControl = dump.isPrivate
? "private, max-age=31536000, immutable"
: "public, max-age=31536000, immutable";
// A private dump's thumbnail must not sit in a shared cache. The app's URLs
// carry the file's version (see dumpFileVersion): what's generated from the
// file is cached for good under the current one, and revalidated otherwise.
const scope = dump.isPrivate ? "private" : "public";
const generatedCache =
ctx.request.url.searchParams.get("v") === dumpFileVersion(dump)
? `${scope}, max-age=31536000, immutable`
: `${scope}, no-cache`;
const customThumb = getDumpThumbnailInfo(dumpId);
if (customThumb) {
// Replaceable without the URL changing: always revalidated (a 304 when
// unchanged).
const served = await serveUploadedFile(
ctx,
`${THUMBNAILS_DIR}/${customThumb.id}-custom`,
customThumb.mime,
3600,
0,
dump.isPrivate,
);
if (served) return;
}
if (dump.kind !== "file" || !dump.fileMime?.startsWith("video/")) {
throw new APIException(
APIErrorCode.NOT_FOUND,
404,
"No video for this dump",
);
const mime = dump.kind === "file" ? dump.fileMime ?? "" : "";
const srcPath = `${DUMPS_DIR}/${dump.id}`;
if (mime.startsWith("video/")) {
const thumbPath = `${THUMBNAILS_DIR}/${dump.id}.jpg`;
if (
!await ensureGenerated(thumbPath, () => videoStill(srcPath, thumbPath))
) {
throw new APIException(
APIErrorCode.NOT_FOUND,
404,
"Could not generate thumbnail",
);
}
if (
await serveFile(ctx, thumbPath, {
mime: "image/jpeg",
cacheControl: generatedCache,
})
) return;
}
const thumbPath = `${THUMBNAILS_DIR}/${dumpId}.jpg`;
// Serve cached thumbnail if it exists
try {
const data = await Deno.readFile(thumbPath);
ctx.response.headers.set("Content-Type", "image/jpeg");
ctx.response.headers.set("Cache-Control", cacheControl);
ctx.response.body = data;
return;
} catch {
// Not cached yet — generate it
if (mime.startsWith("image/")) {
// A scaled-down copy for the feed; the original when there's none to make
// (a GIF, no ffmpeg, an image ffmpeg can't read).
ctx.response.headers.set("X-Content-Type-Options", "nosniff");
const ext = PREVIEWABLE_IMAGES[mime];
if (ext) {
const previewPath = `${THUMBNAILS_DIR}/${dump.id}.preview.${ext}`;
if (
await ensureGenerated(
previewPath,
() => imagePreview(srcPath, previewPath),
) &&
await serveFile(ctx, previewPath, {
mime: ext === "png" ? "image/png" : "image/jpeg",
cacheControl: generatedCache,
})
) return;
}
if (
/^image\/(png|jpeg|gif|webp|avif|bmp)$/.test(mime) &&
await serveFile(ctx, srcPath, { mime, cacheControl: generatedCache })
) return;
}
if (!await ffmpegAvailable()) {
throw new APIException(
APIErrorCode.NOT_FOUND,
404,
"Thumbnail generation unavailable",
);
}
const srcPath = `${DUMPS_DIR}/${dumpId}`;
try {
await generateThumbnail(srcPath, thumbPath);
} catch {
throw new APIException(
APIErrorCode.NOT_FOUND,
404,
"Could not generate thumbnail",
);
}
const data = await Deno.readFile(thumbPath);
ctx.response.headers.set("Content-Type", "image/jpeg");
ctx.response.headers.set("Cache-Control", cacheControl);
ctx.response.body = data;
throw new APIException(
APIErrorCode.NOT_FOUND,
404,
"No thumbnail for this dump",
);
});
export default router;