Files
gerbeur/api/services/youtube-playlist-service.ts
khannurien de5abeceb2
All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 49s
v3: a performance pass over both halves — indexed queries, streamed files, cached previews, and feeds that stop re-rendering on every tick — a review turned up whole-file reads, unindexed scans and render storms; each fix comes with a test. Database: migration 0016 adds the indexes the hot queries lacked — dumps by created_at, (user_id, created_at) and slug, votes by (user_id, created_at), playlists by slug, users by lower(username) for @mentions, and the child side of every cascading foreign key (comment replies, a user's comments, follows both ways, chat messages, invites, reset tokens); the feeds, dump and playlist pages, upvotes and follower fan-out all plan as index searches now, with no temp-table sort. SQLite runs in WAL with synchronous=NORMAL and a busy timeout, and a transaction() helper wraps the multi-row writes (a dump and its categories, category sets, backlinks, attachment links, playlist reorders, follower notifications) so they commit once instead of once a row. Lists no longer run a COUNT(*) per page: they fetch limit + 1 rows and hasMore comes from the extra one; total stays only on search, the one place it's shown. Files: /api/files read the whole upload (up to 50 MB) for every range request a player makes while seeking; api/lib/serve-file.ts streams the requested bytes from disk, with an ETag and 304s, and serves avatars, attachments, covers and thumbnails too. File and thumbnail URLs carry the file's version (size and last change), so the current one is cached for a year and anything else revalidates — a same-size replacement used to stay stale, and a video's still was cached immutable forever under a URL that never changed. Feeds showed image dumps at full size: /api/thumbnails now makes a 960px preview with ffmpeg (GIFs and unreadable images go out as they are), one run per output however many cards ask at once, and ffmpeg is probed once per process instead of spawned per request. Uploads stream to disk; replacing a file writes beside it and renames over, a hard link keeping the old one for rollback instead of holding both in memory; generated stills and previews go when the file is replaced or the dump deleted, and a custom thumbnail with its dump. Outbound: fetchWithTimeout's deadline now covers the body — a server that answered at once and then dripped could hold a create open indefinitely — and pages are read to 4 MB at most; metadata is cached five minutes so the create form's preview and the post that follows share one fetch (refresh-metadata still goes to the page); proxied images sit in a 32 MB LRU. Broadcasts serialize once rather than per socket, and index.html is kept in memory until the file changes. Frontend: currentTime lived in PlayerContext, so every card re-rendered — and re-parsed its Markdown — four times a second while anything played; the playhead has its own PlayerTimeContext now. The socket context is split three ways (feed, chat via useWSChat, presence via useWSPresence), so typing, presence and chat messages leave the feeds alone. DumpCard, JournalCard and Markdown are memoized; card images use the server preview and load lazily; the feed cache is written when the page is left rather than after every pause in scrolling; the theme context value is memoized.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-27 16:35:51 +00:00

217 lines
6.8 KiB
TypeScript

import { APIErrorCode, APIException } from "../model/interfaces.ts";
import { readTextCapped } from "./rich-content-service.ts";
/**
* Resolves a YouTube playlist to its tracklist, at play time.
*
* Playback itself stays in the iframe embed — YouTube plays the list and
* advances through it on its own. This only supplies what the embed doesn't
* expose: every entry's title and length, so the global player can draw a
* tracklist the way it does for a Bandcamp album. The client syncs the
* highlighted row and row clicks through the IFrame Player API.
*
* The data comes from the `ytInitialData` blob on the public playlist page. Its
* shape is undocumented and changes, so the parse is best-effort and accepts
* both the current `lockupViewModel` rows and the older `playlistVideoRenderer`
* ones. Only the first page (100 entries) is read; the client appends any
* further IDs the player reports, untitled.
*/
export interface YouTubePlaylistTrack {
videoId: string;
title: string;
/** Seconds. */
duration?: number;
}
export interface YouTubePlaylist {
listId: string;
title?: string;
tracks: YouTubePlaylistTrack[];
}
/** Titles and lengths rarely change; a stale entry only mislabels a row. */
const TTL_MS = 6 * 60 * 60 * 1000;
const MAX_ENTRIES = 200;
const cache = new Map<string, { value: YouTubePlaylist; at: number }>();
/** Playlist IDs are URL-safe base64-ish: PL…, OLAK5uy_…, RD… (mixes). */
const LIST_ID_RE = /^[\w-]{2,64}$/;
const HEADERS = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
"Accept": "text/html",
"Accept-Language": "en-US,en;q=0.8",
// Pre-answered consent: without it EU requests are redirected to
// consent.youtube.com and never see the playlist.
"Cookie": "SOCS=CAI",
};
/** "3:55", "1:02:03" → seconds. */
function parseClock(text: string): number | undefined {
if (!/^\d+(?::\d{1,2}){1,2}$/.test(text)) return undefined;
return text.split(":").reduce((acc, part) => acc * 60 + Number(part), 0);
}
/** Depth-first walk collecting every value stored under `key`. */
function collect(node: unknown, key: string, out: unknown[]): void {
if (Array.isArray(node)) {
for (const child of node) collect(child, key, out);
} else if (node && typeof node === "object") {
for (const [k, v] of Object.entries(node)) {
if (k === key) out.push(v);
else collect(v, key, out);
}
}
}
function firstString(node: unknown, key: string): string | undefined {
const found: unknown[] = [];
collect(node, key, found);
return found.find((v): v is string => typeof v === "string");
}
/** Follows `path` into parsed JSON; undefined wherever a step is missing. */
function at(node: unknown, ...path: (string | number)[]): unknown {
let cur = node;
for (const key of path) {
if (cur === null || typeof cur !== "object") return undefined;
cur = (cur as Record<string | number, unknown>)[key];
}
return cur;
}
function fromLockup(lockup: unknown): YouTubePlaylistTrack | null {
if (at(lockup, "contentType") !== "LOCKUP_CONTENT_TYPE_VIDEO") return null;
const videoId = at(lockup, "contentId");
const title = at(
lockup,
"metadata",
"lockupMetadataViewModel",
"title",
"content",
);
if (typeof videoId !== "string" || typeof title !== "string") return null;
const badges: unknown[] = [];
collect(at(lockup, "contentImage"), "thumbnailBadgeViewModel", badges);
const duration = badges
.map((b) => at(b, "text"))
.filter((t): t is string => typeof t === "string")
.map(parseClock)
.find((d) => d !== undefined);
return { videoId, title, duration };
}
function fromLegacyRenderer(r: unknown): YouTubePlaylistTrack | null {
const videoId = at(r, "videoId");
const title = at(r, "title", "runs", 0, "text") ??
at(r, "title", "simpleText");
if (typeof videoId !== "string" || typeof title !== "string") return null;
const secs = Number(at(r, "lengthSeconds"));
return {
videoId,
title,
duration: Number.isFinite(secs) && secs > 0 ? secs : undefined,
};
}
export function parsePlaylistPage(
html: string,
listId: string,
): YouTubePlaylist | null {
const start = html.indexOf("var ytInitialData = ");
if (start < 0) return null;
const end = html.indexOf(";</script>", start);
if (end < 0) return null;
let data: unknown;
try {
data = JSON.parse(html.slice(start + "var ytInitialData = ".length, end));
} catch {
return null;
}
const lockups: unknown[] = [];
collect(at(data, "contents"), "lockupViewModel", lockups);
const legacy: unknown[] = [];
collect(at(data, "contents"), "playlistVideoRenderer", legacy);
// A video can appear in a playlist twice; both rows are kept, so the list
// stays positional and matches what the embed plays.
const tracks = [
...lockups.map(fromLockup),
...legacy.map(fromLegacyRenderer),
].filter((t): t is YouTubePlaylistTrack => t !== null);
if (tracks.length === 0) return null;
const metaTitle = at(data, "metadata", "playlistMetadataRenderer", "title");
const title = typeof metaTitle === "string"
? metaTitle
: firstString(at(data, "header"), "content");
return { listId, title, tracks };
}
export async function resolveYouTubePlaylist(
listId: string,
): Promise<YouTubePlaylist> {
if (!LIST_ID_RE.test(listId)) {
throw new APIException(
APIErrorCode.VALIDATION_ERROR,
400,
"Invalid playlist ID",
);
}
const hit = cache.get(listId);
if (hit && Date.now() - hit.at < TTL_MS) return hit.value;
if (hit) cache.delete(listId);
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 8000);
let html: string;
try {
const res = await fetch(
`https://www.youtube.com/playlist?list=${encodeURIComponent(listId)}`,
{ headers: HEADERS, signal: controller.signal },
);
if (!res.ok) {
await res.body?.cancel();
throw new APIException(
APIErrorCode.SERVER_ERROR,
502,
`YouTube returned ${res.status}`,
);
}
html = await readTextCapped(res);
} catch (err) {
if (err instanceof APIException) throw err;
throw new APIException(
APIErrorCode.SERVER_ERROR,
502,
"Could not reach YouTube",
);
} finally {
clearTimeout(timer);
}
const parsed = parsePlaylistPage(html, listId);
if (!parsed) {
// Private, deleted, or auto-generated mixes (RD…) that have no playlist
// page — the client just shows the embed without a tracklist.
throw new APIException(
APIErrorCode.NOT_FOUND,
404,
"No tracks on this playlist",
);
}
if (cache.size >= MAX_ENTRIES) {
const oldest = cache.keys().next().value;
if (oldest !== undefined) cache.delete(oldest);
}
cache.set(listId, { value: parsed, at: Date.now() });
return parsed;
}