v3: a performance pass over both halves — indexed queries, streamed files, cached previews, and feeds that stop re-rendering on every tick — a review turned up whole-file reads, unindexed scans and render storms; each fix comes with a test. Database: migration 0016 adds the indexes the hot queries lacked — dumps by created_at, (user_id, created_at) and slug, votes by (user_id, created_at), playlists by slug, users by lower(username) for @mentions, and the child side of every cascading foreign key (comment replies, a user's comments, follows both ways, chat messages, invites, reset tokens); the feeds, dump and playlist pages, upvotes and follower fan-out all plan as index searches now, with no temp-table sort. SQLite runs in WAL with synchronous=NORMAL and a busy timeout, and a transaction() helper wraps the multi-row writes (a dump and its categories, category sets, backlinks, attachment links, playlist reorders, follower notifications) so they commit once instead of once a row. Lists no longer run a COUNT(*) per page: they fetch limit + 1 rows and hasMore comes from the extra one; total stays only on search, the one place it's shown. Files: /api/files read the whole upload (up to 50 MB) for every range request a player makes while seeking; api/lib/serve-file.ts streams the requested bytes from disk, with an ETag and 304s, and serves avatars, attachments, covers and thumbnails too. File and thumbnail URLs carry the file's version (size and last change), so the current one is cached for a year and anything else revalidates — a same-size replacement used to stay stale, and a video's still was cached immutable forever under a URL that never changed. Feeds showed image dumps at full size: /api/thumbnails now makes a 960px preview with ffmpeg (GIFs and unreadable images go out as they are), one run per output however many cards ask at once, and ffmpeg is probed once per process instead of spawned per request. Uploads stream to disk; replacing a file writes beside it and renames over, a hard link keeping the old one for rollback instead of holding both in memory; generated stills and previews go when the file is replaced or the dump deleted, and a custom thumbnail with its dump. Outbound: fetchWithTimeout's deadline now covers the body — a server that answered at once and then dripped could hold a create open indefinitely — and pages are read to 4 MB at most; metadata is cached five minutes so the create form's preview and the post that follows share one fetch (refresh-metadata still goes to the page); proxied images sit in a 32 MB LRU. Broadcasts serialize once rather than per socket, and index.html is kept in memory until the file changes. Frontend: currentTime lived in PlayerContext, so every card re-rendered — and re-parsed its Markdown — four times a second while anything played; the playhead has its own PlayerTimeContext now. The socket context is split three ways (feed, chat via useWSChat, presence via useWSPresence), so typing, presence and chat messages leave the feeds alone. DumpCard, JournalCard and Markdown are memoized; card images use the server preview and load lazily; the feed cache is written when the page is left rather than after every pause in scrolling; the theme context value is memoized.
All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 49s

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
khannurien
2026-09-27 16:35:51 +00:00
parent 364c498a24
commit de5abeceb2
74 changed files with 1795 additions and 792 deletions

View File

@@ -1,4 +1,4 @@
import { db } from "../model/db.ts";
import { db, transaction } from "../model/db.ts";
import { APIErrorCode, APIException } from "../model/interfaces.ts";
export function createAttachment(id: string, mime: string): void {
@@ -34,9 +34,11 @@ const ATTACHMENT_RE =
*/
export function linkAttachments(text: string, resourceId: string): void {
const ids = [...text.matchAll(ATTACHMENT_RE)].map((m) => m[1]);
for (const id of ids) {
db.prepare(
"UPDATE attachments SET resource_id = ? WHERE id = ? AND resource_id IS NULL",
).run(resourceId, id);
}
if (ids.length === 0) return;
const update = db.prepare(
"UPDATE attachments SET resource_id = ? WHERE id = ? AND resource_id IS NULL",
);
transaction(() => {
for (const id of ids) update.run(resourceId, id);
});
}

View File

@@ -4,6 +4,7 @@ import {
DUMP_SELECT_COLUMNS as SELECT_COLS,
dumpRowToApi,
isDumpRow,
transaction,
} from "../model/db.ts";
import { UUID_RE } from "../lib/slugify.ts";
@@ -78,23 +79,19 @@ export function relinkBacklinks(
targetIds.delete(fromDumpId); // no self-links
db.exec("BEGIN;");
try {
transaction(() => {
db.prepare(
`DELETE FROM dump_backlinks WHERE source_type = ? AND source_id = ?;`,
).run(sourceType, sourceId);
const createdAt = new Date().toISOString();
const insert = db.prepare(
`INSERT INTO dump_backlinks (source_type, source_id, from_dump_id, to_dump_id, created_at)
VALUES (?, ?, ?, ?, ?);`,
);
for (const toDumpId of targetIds) {
db.prepare(
`INSERT INTO dump_backlinks (source_type, source_id, from_dump_id, to_dump_id, created_at)
VALUES (?, ?, ?, ?, ?);`,
).run(sourceType, sourceId, fromDumpId, toDumpId, createdAt);
insert.run(sourceType, sourceId, fromDumpId, toDumpId, createdAt);
}
db.exec("COMMIT;");
} catch (err) {
db.exec("ROLLBACK;");
throw err;
}
});
}
/** Bidirectional, privacy-filtered, capped at 20 — backlinks are a small, bounded list. */

View File

@@ -1,5 +1,9 @@
import { APIErrorCode, APIException } from "../model/interfaces.ts";
import { fetchWithTimeout, isValidHttpUrl } from "./rich-content-service.ts";
import {
fetchWithTimeout,
isValidHttpUrl,
readTextCapped,
} from "./rich-content-service.ts";
import {
BANDCAMP_HOST_RE,
parseTralbum,
@@ -121,7 +125,7 @@ export async function resolveBandcamp(
);
}
const parsed = parseTralbum(await res.text(), pageUrl);
const parsed = parseTralbum(await readTextCapped(res), pageUrl);
if (!parsed) {
// Covers "no data-tralbum", "hasAudio: false" and preorder-only releases
// with nothing streamable — all cases where the client falls back to the

View File

@@ -13,7 +13,9 @@ import {
dumpRowToApi,
isCategoryRow,
isDumpRow,
transaction,
} from "../model/db.ts";
import { type Page, pageOf } from "../lib/pagination.ts";
import { isReservedSlug, isValidSlugFormat } from "../lib/reserved-slugs.ts";
import { VALIDATION } from "../config.ts";
@@ -217,13 +219,15 @@ export function setDumpCategories(
assertCategoriesExist(categoryIds);
const unique = [...new Set(categoryIds)];
db.prepare(`DELETE FROM dump_categories WHERE dump_id = ?;`).run(dumpId);
const insert = db.prepare(
`INSERT INTO dump_categories (dump_id, category_id) VALUES (?, ?);`,
);
for (const categoryId of unique) {
insert.run(dumpId, categoryId);
}
transaction(() => {
db.prepare(`DELETE FROM dump_categories WHERE dump_id = ?;`).run(dumpId);
const insert = db.prepare(
`INSERT INTO dump_categories (dump_id, category_id) VALUES (?, ?);`,
);
for (const categoryId of unique) {
insert.run(dumpId, categoryId);
}
});
}
/** Fill `categoryIds` on each dump with a single batched lookup. */
@@ -251,7 +255,7 @@ export function listDumpsByCategory(
page: number,
limit: number,
requestingUserId?: string,
): { items: Dump[]; total: number } {
): Page<Dump> {
const category = getCategoryBySlug(slug);
const offset = (page - 1) * limit;
@@ -266,16 +270,9 @@ export function listDumpsByCategory(
INNER JOIN dump_categories dc ON dc.dump_id = d.id
WHERE dc.category_id = ? AND ${privacyClause}
ORDER BY d.created_at DESC LIMIT ? OFFSET ?;`,
).all(category.id, ...privacyParams, limit, offset);
const totalRow = db.prepare(
`SELECT COUNT(*) as count
FROM dumps d
INNER JOIN dump_categories dc ON dc.dump_id = d.id
WHERE dc.category_id = ? AND ${privacyClause};`,
).get(category.id, ...privacyParams) as { count: number } | undefined;
).all(category.id, ...privacyParams, limit + 1, offset);
const items = rows.filter(isDumpRow).map(dumpRowToApi);
attachCategoryIds(items);
return { items, total: totalRow?.count ?? 0 };
return pageOf(items, limit);
}

View File

@@ -1,5 +1,6 @@
import { APIErrorCode, APIException, type User } from "../model/interfaces.ts";
import { db, isUserRow, userRowToApi } from "../model/db.ts";
import { type Page, pageOf } from "../lib/pagination.ts";
import { notifyCommentOwnerLike } from "./notification-service.ts";
export interface LikeResult {
@@ -105,13 +106,9 @@ export function getCommentLikers(
commentId: string,
page: number,
limit: number,
): { items: User[]; total: number } {
): Page<User> {
const offset = (page - 1) * limit;
const totalRow = db.prepare(
`SELECT COUNT(*) as count FROM comment_likes WHERE comment_id = ?;`,
).get(commentId) as { count: number } | undefined;
const rawRows = db.prepare(
`SELECT u.id, u.username, u.password_hash, u.role, u.created_at, u.updated_at,
u.avatar_mime, u.description, u.invited_by, u.email,
@@ -121,7 +118,7 @@ export function getCommentLikers(
INNER JOIN comment_likes cl ON cl.comment_id = ? AND cl.user_id = u.id
ORDER BY cl.created_at DESC
LIMIT ? OFFSET ?;`,
).all(commentId, limit, offset);
).all(commentId, limit + 1, offset);
if (!rawRows.every(isUserRow)) {
throw new APIException(
@@ -131,5 +128,5 @@ export function getCommentLikers(
);
}
return { items: rawRows.map(userRowToApi), total: totalRow?.count ?? 0 };
return pageOf(rawRows.map(userRowToApi), limit);
}

View File

@@ -13,7 +13,9 @@ import {
dumpApiToRow,
dumpRowToApi,
isDumpRow,
transaction,
} from "../model/db.ts";
import { type Page, pageOf } from "../lib/pagination.ts";
import {
fetchRichContent,
isValidHttpUrl,
@@ -97,25 +99,26 @@ export async function createUrlDump(
const isPrivate = request.isPrivate ?? false;
const slug = makeSlug(title, dumpId);
db.prepare(
`INSERT INTO dumps (id, kind, title, slug, comment, user_id, created_at, url, url_canonical, rich_content, is_private)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`,
).run(
dumpId,
"url",
title,
slug,
request.comment ?? null,
userId,
createdAt.toISOString(),
request.url,
canonicalizeUrl(request.url),
richContent ? JSON.stringify(richContent) : null,
isPrivate ? 1 : 0,
);
const categoryIds = [...new Set(request.categoryIds ?? [])];
setDumpCategories(dumpId, categoryIds);
transaction(() => {
db.prepare(
`INSERT INTO dumps (id, kind, title, slug, comment, user_id, created_at, url, url_canonical, rich_content, is_private)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`,
).run(
dumpId,
"url",
title,
slug,
request.comment ?? null,
userId,
createdAt.toISOString(),
request.url,
canonicalizeUrl(request.url),
richContent ? JSON.stringify(richContent) : null,
isPrivate ? 1 : 0,
);
setDumpCategories(dumpId, categoryIds);
});
const dump: Dump = {
id: dumpId,
@@ -176,28 +179,30 @@ export async function createFileDump(
const dedupedCategoryIds = [...new Set(categoryIds)];
await Deno.mkdir(DUMPS_DIR, { recursive: true });
const data = new Uint8Array(await file.arrayBuffer());
try {
await Deno.writeFile(`${DUMPS_DIR}/${dumpId}`, data);
// Streamed to disk rather than copied into one more 50 MB buffer first.
await Deno.writeFile(`${DUMPS_DIR}/${dumpId}`, file.stream());
db.prepare(
`INSERT INTO dumps (id, kind, title, slug, comment, user_id, created_at, file_name, file_mime, file_size, is_private)
transaction(() => {
db.prepare(
`INSERT INTO dumps (id, kind, title, slug, comment, user_id, created_at, file_name, file_mime, file_size, is_private)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`,
).run(
dumpId,
"file",
finalTitle,
slug,
comment ?? null,
userId,
createdAt.toISOString(),
file.name,
file.type,
file.size,
isPrivate ? 1 : 0,
);
setDumpCategories(dumpId, dedupedCategoryIds);
).run(
dumpId,
"file",
finalTitle,
slug,
comment ?? null,
userId,
createdAt.toISOString(),
file.name,
file.type,
file.size,
isPrivate ? 1 : 0,
);
setDumpCategories(dumpId, dedupedCategoryIds);
});
} catch (err) {
// Roll back the file if DB insert fails
await Deno.remove(`${DUMPS_DIR}/${dumpId}`).catch(() => {});
@@ -232,6 +237,29 @@ export async function createFileDump(
return dump;
}
/**
* A file dump's version as the app puts it in file and thumbnail URLs
* (`?v=`): it changes whenever the file can have. Mirrored by
* `dumpFileVersion` in src/utils/urls.ts.
*/
export function dumpFileVersion(
dump: Pick<Dump, "fileSize" | "createdAt" | "updatedAt">,
): string {
return `${dump.fileSize ?? 0}-${
(dump.updatedAt ?? dump.createdAt).getTime()
}`;
}
/** Files generated from a dump's upload (see api/routes/thumbnails.ts):
* dropped when it's replaced or gone, so they're made again from the new one. */
async function removeGeneratedThumbnails(dumpId: string): Promise<void> {
await Promise.all(
[`${dumpId}.jpg`, `${dumpId}.preview.jpg`, `${dumpId}.preview.png`].map(
(name) => Deno.remove(`${THUMBNAILS_DIR}/${name}`).catch(() => {}),
),
);
}
// Internal fetch — no privacy check. Use only when ownership is already enforced.
function fetchDump(idOrSlug: string): Dump {
const row = UUID_RE.test(idOrSlug)
@@ -296,23 +324,16 @@ export function listDumps(
page: number,
limit: number,
requestingUserId?: string,
): { items: Dump[]; total: number } {
): Page<Dump> {
const offset = (page - 1) * limit;
// Show public dumps + the requesting user's own private dumps
const rows = requestingUserId
? db.prepare(
`SELECT ${SELECT_COLS} FROM dumps WHERE (is_private = 0 OR user_id = ?) ORDER BY created_at DESC LIMIT ? OFFSET ?;`,
).all(requestingUserId, limit, offset)
).all(requestingUserId, limit + 1, offset)
: db.prepare(
`SELECT ${SELECT_COLS} FROM dumps WHERE is_private = 0 ORDER BY created_at DESC LIMIT ? OFFSET ?;`,
).all(limit, offset);
const totalRow = requestingUserId
? db.prepare(
`SELECT COUNT(*) as count FROM dumps WHERE (is_private = 0 OR user_id = ?);`,
).get(requestingUserId) as { count: number } | undefined
: db.prepare(
`SELECT COUNT(*) as count FROM dumps WHERE is_private = 0;`,
).get() as { count: number } | undefined;
).all(limit + 1, offset);
if (!rows || !rows.every(isDumpRow)) {
throw new APIException(
@@ -324,7 +345,7 @@ export function listDumps(
const items = rows.map(dumpRowToApi);
attachCategoryIds(items);
return { items, total: totalRow?.count ?? 0 };
return pageOf(items, limit);
}
/**
@@ -548,11 +569,24 @@ export async function replaceFileDump(
}
if (categoryIds !== undefined) assertCategoriesExist(categoryIds);
const data = new Uint8Array(await file.arrayBuffer());
// The new file is written beside the old one and renamed over it (atomic: a
// reader sees one or the other). A hard link keeps the old one reachable so
// a failed update can put it back, without holding either in memory.
const filePath = `${DUMPS_DIR}/${dumpId}`;
// Read old file contents so we can restore on DB failure
const oldData = await Deno.readFile(filePath).catch(() => null);
await Deno.writeFile(filePath, data);
const tmpPath = `${filePath}.${crypto.randomUUID()}.tmp`;
const backupPath = `${tmpPath}.old`;
await Deno.mkdir(DUMPS_DIR, { recursive: true });
try {
await Deno.writeFile(tmpPath, file.stream());
} catch (err) {
await Deno.remove(tmpPath).catch(() => {});
throw err;
}
const hadOld = await Deno.link(filePath, backupPath).then(
() => true,
() => false,
);
await Deno.rename(tmpPath, filePath);
const now = new Date();
const finalTitle = title?.trim() || dump.title;
@@ -561,30 +595,35 @@ export async function replaceFileDump(
: dump.slug;
const finalPrivate = isPrivate ?? dump.isPrivate;
try {
db.prepare(
`UPDATE dumps SET title = ?, slug = ?, file_name = ?, file_mime = ?, file_size = ?, comment = ?, is_private = ?, updated_at = ? WHERE id = ?;`,
).run(
finalTitle,
newSlug ?? null,
file.name,
file.type,
file.size,
comment ?? null,
finalPrivate ? 1 : 0,
now.toISOString(),
dumpId,
);
transaction(() => {
db.prepare(
`UPDATE dumps SET title = ?, slug = ?, file_name = ?, file_mime = ?, file_size = ?, comment = ?, is_private = ?, updated_at = ? WHERE id = ?;`,
).run(
finalTitle,
newSlug ?? null,
file.name,
file.type,
file.size,
comment ?? null,
finalPrivate ? 1 : 0,
now.toISOString(),
dumpId,
);
if (categoryIds !== undefined) {
setDumpCategories(dumpId, [...new Set(categoryIds)]);
}
});
} catch (err) {
// Roll back the file to its previous contents on DB failure
if (oldData) await Deno.writeFile(filePath, oldData).catch(() => {});
// Put the previous file back on DB failure
if (hadOld) await Deno.rename(backupPath, filePath).catch(() => {});
else await Deno.remove(filePath).catch(() => {});
throw err;
}
if (hadOld) await Deno.remove(backupPath).catch(() => {});
await removeGeneratedThumbnails(dumpId);
if (categoryIds !== undefined) {
const ids = [...new Set(categoryIds)];
setDumpCategories(dumpId, ids);
dump.categoryIds = ids;
dump.categoryIds = [...new Set(categoryIds)];
} else {
attachCategoryIds([dump]);
}
@@ -616,15 +655,12 @@ export function getDumpsByUser(
page: number,
limit: number,
includePrivate: boolean,
): { items: Dump[]; total: number } {
): Page<Dump> {
const offset = (page - 1) * limit;
const privacyFilter = includePrivate ? "" : " AND is_private = 0";
const rows = db.prepare(
`SELECT ${SELECT_COLS} FROM dumps WHERE user_id = ?${privacyFilter} ORDER BY created_at DESC LIMIT ? OFFSET ?;`,
).all(userId, limit, offset);
const totalRow = db.prepare(
`SELECT COUNT(*) as count FROM dumps WHERE user_id = ?${privacyFilter};`,
).get(userId) as { count: number } | undefined;
).all(userId, limit + 1, offset);
if (!rows.every(isDumpRow)) {
throw new APIException(
APIErrorCode.SERVER_ERROR,
@@ -634,7 +670,7 @@ export function getDumpsByUser(
}
const items = rows.map(dumpRowToApi);
attachCategoryIds(items);
return { items, total: totalRow?.count ?? 0 };
return pageOf(items, limit);
}
export function getVotedDumpsByUser(
@@ -642,11 +678,9 @@ export function getVotedDumpsByUser(
page: number,
limit: number,
requestingUserId: string | null,
): { items: Dump[]; total: number } {
): Page<Dump> {
const offset = (page - 1) * limit;
const dumpCols = SELECT_COLS_ALIASED;
let totalRow: { count: number } | undefined;
let rows: unknown[];
if (requestingUserId === userId) {
@@ -657,12 +691,7 @@ export function getVotedDumpsByUser(
INNER JOIN votes v ON d.id = v.dump_id
WHERE v.user_id = ? AND (d.is_private = 0 OR d.user_id = ?)
ORDER BY v.created_at DESC LIMIT ? OFFSET ?;`,
).all(userId, userId, limit, offset);
totalRow = db.prepare(
`SELECT COUNT(*) as count FROM dumps d
INNER JOIN votes v ON d.id = v.dump_id
WHERE v.user_id = ? AND (d.is_private = 0 OR d.user_id = ?);`,
).get(userId, userId) as { count: number } | undefined;
).all(userId, userId, limit + 1, offset);
} else {
rows = db.prepare(
`SELECT ${dumpCols}
@@ -670,12 +699,7 @@ export function getVotedDumpsByUser(
INNER JOIN votes v ON d.id = v.dump_id
WHERE v.user_id = ? AND d.is_private = 0
ORDER BY v.created_at DESC LIMIT ? OFFSET ?;`,
).all(userId, limit, offset);
totalRow = db.prepare(
`SELECT COUNT(*) as count FROM dumps d
INNER JOIN votes v ON d.id = v.dump_id
WHERE v.user_id = ? AND d.is_private = 0;`,
).get(userId) as { count: number } | undefined;
).all(userId, limit + 1, offset);
}
if (!rows.every(isDumpRow)) {
@@ -687,7 +711,7 @@ export function getVotedDumpsByUser(
}
const items = rows.map(dumpRowToApi);
attachCategoryIds(items);
return { items, total: totalRow?.count ?? 0 };
return pageOf(items, limit);
}
export async function refreshDumpMetadata(dumpId: string): Promise<Dump> {
@@ -705,7 +729,9 @@ export async function refreshDumpMetadata(dumpId: string): Promise<Dump> {
// over a dump that already has good metadata would silently destroy it and
// rename the dump to its bare hostname, so bail out instead — the caller
// surfaces the error and nothing is lost.
const { ok, content: richContent } = await tryFetchRichContent(dump.url);
const { ok, content: richContent } = await tryFetchRichContent(dump.url, {
fresh: true,
});
if (!ok) {
throw new APIException(
APIErrorCode.SERVER_ERROR,
@@ -767,7 +793,11 @@ export async function deleteDump(dumpId: string): Promise<void> {
}
if (dump.kind === "file") {
await Deno.remove(`${DUMPS_DIR}/${dumpId}`).catch(() => {});
await Deno.remove(`${DUMPS_DIR}/${dump.id}`).catch(() => {});
await removeGeneratedThumbnails(dump.id);
}
if (dump.thumbnailMime) {
await Deno.remove(`${THUMBNAILS_DIR}/${dump.id}-custom`).catch(() => {});
}
broadcastDumpDeleted(dumpId);

View File

@@ -21,6 +21,7 @@ import {
playlistRowToApi,
userRowToApi,
} from "../model/db.ts";
import { type Page, pageOf } from "../lib/pagination.ts";
import { attachCategoryIds } from "./category-service.ts";
// ── Follow / unfollow a user ──────────────────────────────────────────────────
@@ -141,7 +142,7 @@ export function getFollowedUsersDumpFeed(
page: number,
limit: number,
categoryId?: string,
): { items: Dump[]; total: number } {
): Page<Dump> {
const offset = (page - 1) * limit;
// Optional category filter: join the membership table and constrain to it.
@@ -159,16 +160,7 @@ export function getFollowedUsersDumpFeed(
AND d.is_private = 0
ORDER BY d.created_at DESC
LIMIT ? OFFSET ?;`,
).all(...catParams, followerId, limit, offset);
const totalRow = db.prepare(
`SELECT COUNT(*) as count
FROM dumps d
INNER JOIN follows f ON f.followed_user_id = d.user_id
${catJoin}
WHERE f.follower_id = ?
AND d.is_private = 0;`,
).get(...catParams, followerId) as { count: number } | undefined;
).all(...catParams, followerId, limit + 1, offset);
const userFeedRows = rawRows as Parameters<typeof isDumpRow>[0][];
if (!userFeedRows.every(isDumpRow)) {
@@ -180,7 +172,7 @@ export function getFollowedUsersDumpFeed(
}
const items = userFeedRows.map(dumpRowToApi);
attachCategoryIds(items);
return { items, total: totalRow?.count ?? 0 };
return pageOf(items, limit);
}
// ── Followed-playlists dump feed ──────────────────────────────────────────────
@@ -190,7 +182,7 @@ export function getFollowedPlaylistsDumpFeed(
page: number,
limit: number,
categoryId?: string,
): { items: Dump[]; total: number } {
): Page<Dump> {
const offset = (page - 1) * limit;
const catJoin = categoryId
@@ -211,19 +203,7 @@ export function getFollowedPlaylistsDumpFeed(
GROUP BY d.id
ORDER BY MAX(pd.added_at) DESC
LIMIT ? OFFSET ?;`,
).all(...catParams, followerId, limit, offset);
const totalRow = db.prepare(
`SELECT COUNT(DISTINCT d.id) as count
FROM dumps d
INNER JOIN playlist_dumps pd ON pd.dump_id = d.id
INNER JOIN playlists p ON p.id = pd.playlist_id
INNER JOIN follows f ON f.followed_playlist_id = p.id
${catJoin}
WHERE f.follower_id = ?
AND p.is_public = 1
AND d.is_private = 0;`,
).get(...catParams, followerId) as { count: number } | undefined;
).all(...catParams, followerId, limit + 1, offset);
if (!rawRows.every(isDumpRow)) {
throw new APIException(
@@ -234,7 +214,7 @@ export function getFollowedPlaylistsDumpFeed(
}
const items = rawRows.map(dumpRowToApi);
attachCategoryIds(items);
return { items, total: totalRow?.count ?? 0 };
return pageOf(items, limit);
}
// ── Followed playlists (as playlist objects) ──────────────────────────────────
@@ -243,15 +223,9 @@ export function getFollowedPlaylistsByUser(
userId: string,
page: number,
limit: number,
): { items: Playlist[]; total: number } {
): Page<Playlist> {
const offset = (page - 1) * limit;
const totalRow = db.prepare(
`SELECT COUNT(*) as count
FROM follows f
WHERE f.follower_id = ? AND f.followed_playlist_id IS NOT NULL;`,
).get(userId) as { count: number } | undefined;
const rawRows = db.prepare(
`SELECT p.*, u.username as owner_username,
(SELECT COUNT(*) FROM playlist_dumps pd WHERE pd.playlist_id = p.id) as dump_count
@@ -262,7 +236,7 @@ export function getFollowedPlaylistsByUser(
AND p.is_public = 1
ORDER BY f.created_at DESC
LIMIT ? OFFSET ?;`,
).all(userId, limit, offset);
).all(userId, limit + 1, offset);
if (!rawRows.every(isPlaylistRow)) {
throw new APIException(
@@ -271,7 +245,7 @@ export function getFollowedPlaylistsByUser(
"Malformed playlist data",
);
}
return { items: rawRows.map(playlistRowToApi), total: totalRow?.count ?? 0 };
return pageOf(rawRows.map(playlistRowToApi), limit);
}
// ── Followed users (as user objects) ─────────────────────────────────────────
@@ -280,13 +254,9 @@ export function getFollowedUsersByUser(
userId: string,
page: number,
limit: number,
): { items: User[]; total: number } {
): Page<User> {
const offset = (page - 1) * limit;
const totalRow = db.prepare(
`SELECT COUNT(*) as count FROM follows WHERE follower_id = ? AND followed_user_id IS NOT NULL;`,
).get(userId) as { count: number } | undefined;
const rawRows = db.prepare(
`SELECT u.id, u.username, u.password_hash, u.role, u.created_at, u.updated_at,
u.avatar_mime, u.description, u.invited_by, u.email,
@@ -297,7 +267,7 @@ export function getFollowedUsersByUser(
WHERE f.follower_id = ?
ORDER BY f.created_at DESC
LIMIT ? OFFSET ?;`,
).all(userId, limit, offset);
).all(userId, limit + 1, offset);
if (!rawRows.every(isUserRow)) {
throw new APIException(
@@ -307,5 +277,5 @@ export function getFollowedUsersByUser(
);
}
return { items: rawRows.map(userRowToApi), total: totalRow?.count ?? 0 };
return pageOf(rawRows.map(userRowToApi), limit);
}

View File

@@ -5,7 +5,13 @@ import type {
UserDumpPostedData,
} from "../model/interfaces.ts";
import { APIErrorCode, APIException } from "../model/interfaces.ts";
import { db, isNotificationRow, notificationRowToApi } from "../model/db.ts";
import {
db,
isNotificationRow,
notificationRowToApi,
transaction,
} from "../model/db.ts";
import { type Page, pageOf } from "../lib/pagination.ts";
import { sendToUser } from "./ws-service.ts";
// Regex: matches @username not already inside a markdown link ([...] or (...)
@@ -54,15 +60,11 @@ export function getNotificationsForUser(
userId: string,
page: number,
limit: number,
): { items: Notification[]; total: number } {
): Page<Notification> {
const offset = (page - 1) * limit;
const rawRows = db.prepare(
`SELECT * FROM notifications WHERE user_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?;`,
).all(userId, limit, offset);
const totalRow = db.prepare(
`SELECT COUNT(*) as count FROM notifications WHERE user_id = ?;`,
).get(userId) as { count: number } | undefined;
).all(userId, limit + 1, offset);
if (!rawRows.every(isNotificationRow)) {
throw new APIException(
@@ -71,10 +73,7 @@ export function getNotificationsForUser(
"Malformed notification data",
);
}
return {
items: rawRows.map(notificationRowToApi),
total: totalRow?.count ?? 0,
};
return pageOf(rawRows.map(notificationRowToApi), limit);
}
export function getUnreadCount(userId: string): number {
@@ -179,30 +178,35 @@ export function notifyUserFollowersNewDump(
VALUES (?, ?, ?, ?, 0, ?, ?);`,
);
for (const row of followerRows) {
const id = crypto.randomUUID();
const result = insert.run(
id,
row.follower_id,
"user_dump_posted",
dataJson,
createdAt,
sourceKey,
);
if ((result.changes as number) > 0) {
sendToUser(row.follower_id, {
type: "notification_created",
notification: {
id,
userId: row.follower_id,
type: "user_dump_posted",
data,
read: false,
createdAt,
},
});
// One transaction for the whole fan-out: a commit per follower is what made
// a popular user's dump slow to post. The live events go out inside it, but
// the commit follows in the same tick, before any client can ask back.
transaction(() => {
for (const row of followerRows) {
const id = crypto.randomUUID();
const result = insert.run(
id,
row.follower_id,
"user_dump_posted",
dataJson,
createdAt,
sourceKey,
);
if ((result.changes as number) > 0) {
sendToUser(row.follower_id, {
type: "notification_created",
notification: {
id,
userId: row.follower_id,
type: "user_dump_posted",
data,
read: false,
createdAt,
},
});
}
}
}
});
}
export function notifyDumpOwnerUpvote(
@@ -350,12 +354,14 @@ export function notifyPlaylistFollowersNewDump(
`SELECT follower_id FROM follows WHERE followed_playlist_id = ?;`,
).all(playlistId) as { follower_id: string }[];
for (const row of followerRows) {
createNotification(
row.follower_id,
"playlist_dump_added",
{ dumpId, dumpTitle, playlistId, playlistTitle },
`pdump:${playlistId}:${dumpId}`,
);
}
transaction(() => {
for (const row of followerRows) {
createNotification(
row.follower_id,
"playlist_dump_added",
{ dumpId, dumpTitle, playlistId, playlistTitle },
`pdump:${playlistId}:${dumpId}`,
);
}
});
}

View File

@@ -15,7 +15,9 @@ import {
isDumpRow,
isPlaylistRow,
playlistRowToApi,
transaction,
} from "../model/db.ts";
import { type Page, pageOf } from "../lib/pagination.ts";
import {
broadcastPlaylistCreated,
broadcastPlaylistDeleted,
@@ -136,25 +138,16 @@ export function listPlaylistsByUser(
requestingUserId: string | null,
page: number,
limit: number,
): { items: Playlist[]; total: number } {
): Page<Playlist> {
const isOwner = requestingUserId === userId;
const offset = (page - 1) * limit;
const countSql = isOwner
? `SELECT COUNT(*) as count FROM playlists WHERE user_id = ?;`
: `SELECT COUNT(*) as count FROM playlists WHERE user_id = ? AND is_public = 1;`;
const sql = isOwner
? `SELECT ${PLAYLIST_SELECT} WHERE p.user_id = ? ORDER BY p.created_at DESC LIMIT ? OFFSET ?;`
: `SELECT ${PLAYLIST_SELECT} WHERE p.user_id = ? AND p.is_public = 1 ORDER BY p.created_at DESC LIMIT ? OFFSET ?;`;
const totalRow = db.prepare(countSql).get(userId) as
| { count: number }
| undefined;
const rows = db.prepare(sql).all(userId, limit, offset);
return {
items: rows.filter(isPlaylistRow).map(playlistRowToApi),
total: totalRow?.count ?? 0,
};
const rows = db.prepare(sql).all(userId, limit + 1, offset);
return pageOf(rows.filter(isPlaylistRow).map(playlistRowToApi), limit);
}
export function updatePlaylist(
@@ -382,9 +375,11 @@ export function reorderPlaylist(
const update = db.prepare(
`UPDATE playlist_dumps SET position = ? WHERE playlist_id = ? AND dump_id = ?;`,
);
for (let i = 0; i < order.length; i++) {
update.run(i, playlist.id, order[i]);
}
transaction(() => {
for (let i = 0; i < order.length; i++) {
update.run(i, playlist.id, order[i]);
}
});
broadcastPlaylistDumpsUpdated(playlist, getCurrentMembers(playlist.id));
}

View File

@@ -1,6 +1,10 @@
import type { RichContent } from "../../model/interfaces.ts";
import type { RichContentProvider } from "../rich-content-service.ts";
import { extractOgTag, fetchWithTimeout } from "../rich-content-service.ts";
import {
extractOgTag,
fetchWithTimeout,
readTextCapped,
} from "../rich-content-service.ts";
const BANDCAMP_REGEX = /(?:^|\.)bandcamp\.com/;
@@ -20,10 +24,11 @@ export const bandcampProvider: RichContentProvider = {
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
return { type: "bandcamp", siteName: "Bandcamp", url };
}
const html = await res.text();
const html = await readTextCapped(res);
return {
type: "bandcamp",

View File

@@ -12,6 +12,7 @@ import {
extractThemeColor,
fetchWithTimeout,
normalizeCssColor,
readTextCapped,
} from "../rich-content-service.ts";
export const genericProvider: RichContentProvider = {
@@ -26,10 +27,11 @@ export const genericProvider: RichContentProvider = {
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
return { type: "generic", url };
}
const html = await res.text();
const html = await readTextCapped(res);
const ld = extractJsonLd(html);
// If og:url is present but points to a different page (e.g. the homepage),

View File

@@ -1,6 +1,10 @@
import type { RichContent } from "../../model/interfaces.ts";
import type { RichContentProvider } from "../rich-content-service.ts";
import { extractOgTag, fetchWithTimeout } from "../rich-content-service.ts";
import {
extractOgTag,
fetchWithTimeout,
readTextCapped,
} from "../rich-content-service.ts";
export const soundcloudProvider: RichContentProvider = {
name: "soundcloud",
@@ -18,10 +22,11 @@ export const soundcloudProvider: RichContentProvider = {
const contentType = res.headers.get("content-type") ?? "";
if (!contentType.startsWith("text/html")) {
await res.body?.cancel();
return { type: "soundcloud", siteName: "SoundCloud", url };
}
const html = await res.text();
const html = await readTextCapped(res);
return {
type: "soundcloud",

View File

@@ -1,6 +1,10 @@
import type { RichContent } from "../../model/interfaces.ts";
import type { RichContentProvider } from "../rich-content-service.ts";
import { extractOgTag, fetchWithTimeout } from "../rich-content-service.ts";
import {
extractOgTag,
fetchWithTimeout,
readTextCapped,
} from "../rich-content-service.ts";
function extractVideoId(url: string): string | null {
try {
@@ -91,6 +95,7 @@ async function fetchOEmbed(
};
return { title: data.title, thumbnailUrl: data.thumbnail_url };
}
await res.body?.cancel();
} catch {
// oembed failed — carry on
}
@@ -121,9 +126,11 @@ export const youtubeProvider: RichContentProvider = {
try {
const res = await fetchWithTimeout(url);
if (res.ok) {
const html = await res.text();
const html = await readTextCapped(res);
title = extractOgTag(html, "title");
thumbnailUrl = extractOgTag(html, "image");
} else {
await res.body?.cancel();
}
} catch {
// scrape failed — carry on with undefined values

View File

@@ -70,6 +70,8 @@ async function fetchViaCurl(
`${url.hostname.replace(/^\[(.*)\]$/, "$1")}:${port}:${pinned}`,
"--max-time",
String(Math.ceil(timeoutMs / 1000)),
"--max-filesize",
String(2 * MAX_PAGE_BYTES),
"--user-agent",
FETCH_HEADERS["User-Agent"],
"--header",
@@ -118,7 +120,78 @@ async function fetchViaCurl(
}
}
/** One request, no redirects followed. */
/**
* Most bytes of a fetched page that are read. Metadata lives in the <head>,
* and a "page" past this is a download, not something to scrape.
*/
export const MAX_PAGE_BYTES = 4 * 1024 * 1024;
/**
* Hands `body` on, and clears the request's deadline once it has been read to
* the end or cancelled. Until then the deadline still applies: a server that
* answers at once and then drips its body can't hold the request open.
*/
function withDeadline(
body: ReadableStream<Uint8Array>,
timer: ReturnType<typeof setTimeout>,
): ReadableStream<Uint8Array> {
const reader = body.getReader();
return new ReadableStream({
async pull(controller) {
try {
const { done, value } = await reader.read();
if (done) {
clearTimeout(timer);
controller.close();
} else {
controller.enqueue(value);
}
} catch (err) {
clearTimeout(timer);
controller.error(err);
}
},
cancel(reason) {
clearTimeout(timer);
return reader.cancel(reason);
},
});
}
/**
* Reads a response's body as text, stopping at `maxBytes` (the rest is
* cancelled, never downloaded).
*/
export async function readTextCapped(
res: Response,
maxBytes = MAX_PAGE_BYTES,
): Promise<string> {
if (!res.body) return "";
const reader = res.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
try {
while (size < maxBytes) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
size += value.byteLength;
}
} finally {
await reader.cancel().catch(() => {});
}
const bytes = new Uint8Array(Math.min(size, maxBytes));
let offset = 0;
for (const chunk of chunks) {
const part = chunk.subarray(0, bytes.byteLength - offset);
bytes.set(part, offset);
offset += part.byteLength;
if (offset >= bytes.byteLength) break;
}
return new TextDecoder().decode(bytes);
}
/** One request, no redirects followed. The timeout covers the body too. */
async function fetchOnce(
url: URL,
address: string,
@@ -126,13 +199,15 @@ async function fetchOnce(
): Promise<Response> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
let res: Response;
try {
return await fetch(url, {
res = await fetch(url, {
signal: controller.signal,
headers: FETCH_HEADERS,
redirect: "manual",
});
} catch (err) {
clearTimeout(timer);
if (!(err instanceof TypeError && err.message.includes("certificate"))) {
throw err;
}
@@ -145,9 +220,16 @@ async function fetchOnce(
if (curlRes) return curlRes;
throw err;
} finally {
clearTimeout(timer);
}
if (!res.body) {
clearTimeout(timer);
return res;
}
return new Response(withDeadline(res.body, timer), {
status: res.status,
statusText: res.statusText,
headers: res.headers,
});
}
/**
@@ -606,7 +688,45 @@ export interface FetchRichContentResult {
* On failure it still yields a minimal stub so a *new* dump has something
* displayable, but `ok: false` lets callers with existing metadata keep it.
*/
export async function tryFetchRichContent(
// Metadata fetched in the last few minutes, by URL. The create form previews a
// link and then posts it; without this the page was fetched twice, and the
// second fetch was the slow part of posting. Only successes are kept, and a
// metadata refresh (`fresh`) always goes to the page.
const RECENT_TTL_MS = 5 * 60 * 1000;
const RECENT_MAX = 200;
const recent = new Map<
string,
{ at: number; result: Promise<FetchRichContentResult> }
>();
/** Forget every cached result (tests). */
export function clearRichContentCache(): void {
recent.clear();
}
export function tryFetchRichContent(
url: string,
{ fresh = false }: { fresh?: boolean } = {},
): Promise<FetchRichContentResult> {
const hit = recent.get(url);
if (!fresh && hit && Date.now() - hit.at < RECENT_TTL_MS) return hit.result;
recent.delete(url);
// Stored while in flight too, so a preview and a post racing share a fetch.
const result = fetchRichContentUncached(url);
if (recent.size >= RECENT_MAX) {
const oldest = recent.keys().next().value;
if (oldest !== undefined) recent.delete(oldest);
}
const entry = { at: Date.now(), result };
recent.set(url, entry);
result.then(({ ok }) => {
if (!ok && recent.get(url) === entry) recent.delete(url);
});
return result;
}
async function fetchRichContentUncached(
url: string,
): Promise<FetchRichContentResult> {
try {

View File

@@ -1,5 +1,6 @@
import { APIErrorCode, APIException, type User } from "../model/interfaces.ts";
import { db, isUserRow, userRowToApi } from "../model/db.ts";
import { type Page, pageOf } from "../lib/pagination.ts";
import { notifyDumpOwnerUpvote } from "./notification-service.ts";
export interface VoteResult {
@@ -98,13 +99,9 @@ export function getDumpVoters(
dumpId: string,
page: number,
limit: number,
): { items: User[]; total: number } {
): Page<User> {
const offset = (page - 1) * limit;
const totalRow = db.prepare(
`SELECT COUNT(*) as count FROM votes WHERE dump_id = ?;`,
).get(dumpId) as { count: number } | undefined;
const rawRows = db.prepare(
`SELECT u.id, u.username, u.password_hash, u.role, u.created_at, u.updated_at,
u.avatar_mime, u.description, u.invited_by, u.email,
@@ -114,7 +111,7 @@ export function getDumpVoters(
INNER JOIN votes v ON v.dump_id = ? AND v.user_id = u.id
ORDER BY v.created_at DESC
LIMIT ? OFFSET ?;`,
).all(dumpId, limit, offset);
).all(dumpId, limit + 1, offset);
if (!rawRows.every(isUserRow)) {
throw new APIException(
@@ -124,5 +121,5 @@ export function getDumpVoters(
);
}
return { items: rawRows.map(userRowToApi), total: totalRow?.count ?? 0 };
return pageOf(rawRows.map(userRowToApi), limit);
}

View File

@@ -64,18 +64,30 @@ export function getOnlineUsers(): OnlineUser[] {
return Array.from(seen.values());
}
function sendRaw(socket: WebSocket, payload: string): void {
if (socket.readyState === WebSocket.OPEN) socket.send(payload);
}
function send(socket: WebSocket, data: ServerToClientMessage): void {
if (socket.readyState === WebSocket.OPEN) {
socket.send(JSON.stringify(data));
sendRaw(socket, JSON.stringify(data));
}
/** Sends one message to every client `include` accepts (all by default),
* serialized once rather than once per socket. */
function broadcast(
data: ServerToClientMessage,
include?: (client: WsClient) => boolean,
): void {
let payload: string | undefined;
for (const client of clients) {
if (include && !include(client)) continue;
payload ??= JSON.stringify(data);
sendRaw(client.socket, payload);
}
}
export function sendToUser(userId: string, data: ServerToClientMessage): void {
for (const client of clients) {
if (client.userId === userId) {
send(client.socket, data);
}
}
broadcast(data, (client) => client.userId === userId);
}
export function disconnectUser(userId: string): void {
@@ -100,28 +112,19 @@ export function closeUserSockets(userId: string, reason: string): void {
}
export function broadcastPresence(): void {
const users = getOnlineUsers();
for (const client of clients) {
send(client.socket, { type: "presence_update", users });
}
broadcast({ type: "presence_update", users: getOnlineUsers() });
}
export function broadcastNewDump(dump: Dump): void {
for (const client of clients) {
send(client.socket, { type: "dump_created", dump });
}
broadcast({ type: "dump_created", dump });
}
export function broadcastDumpUpdated(dump: Dump): void {
for (const client of clients) {
send(client.socket, { type: "dump_updated", dump });
}
broadcast({ type: "dump_updated", dump });
}
export function broadcastDumpDeleted(dumpId: string): void {
for (const client of clients) {
send(client.socket, { type: "dump_deleted", dumpId });
}
broadcast({ type: "dump_deleted", dumpId });
}
// `onlyUserId`: a private dump's count goes to that user's sockets alone.
@@ -132,16 +135,10 @@ export function broadcastVoteUpdate(
action: "cast" | "remove",
onlyUserId?: string,
): void {
for (const client of clients) {
if (onlyUserId && client.userId !== onlyUserId) continue;
send(client.socket, {
type: "votes_update",
dumpId,
voteCount,
voterId,
action,
});
}
broadcast(
{ type: "votes_update", dumpId, voteCount, voterId, action },
onlyUserId ? (client) => client.userId === onlyUserId : undefined,
);
}
// `onlyUserId`: a like on a private dump's comment goes to that user alone.
@@ -152,27 +149,20 @@ export function broadcastCommentLikeUpdate(
action: "cast" | "remove",
onlyUserId?: string,
): void {
for (const client of clients) {
if (onlyUserId && client.userId !== onlyUserId) continue;
send(client.socket, {
type: "comment_likes_update",
commentId,
likeCount,
likerId,
action,
});
}
broadcast(
{ type: "comment_likes_update", commentId, likeCount, likerId, action },
onlyUserId ? (client) => client.userId === onlyUserId : undefined,
);
}
function sendToPlaylistAudience(
playlist: Pick<Playlist, "isPublic" | "userId">,
data: ServerToClientMessage,
): void {
for (const client of clients) {
if (playlist.isPublic || client.userId === playlist.userId) {
send(client.socket, data);
}
}
broadcast(
data,
(client) => playlist.isPublic || client.userId === playlist.userId,
);
}
export function broadcastPlaylistCreated(playlist: Playlist): void {
@@ -224,38 +214,28 @@ export function broadcastPlaylistDumpsUpdated(
}
export function broadcastUserUpdated(user: PublicUser): void {
for (const client of clients) {
send(client.socket, { type: "user_updated", user });
}
broadcast({ type: "user_updated", user });
}
export function broadcastCommentCreated(comment: Comment): void {
for (const client of clients) {
send(client.socket, { type: "comment_created", comment });
}
broadcast({ type: "comment_created", comment });
}
export function broadcastCommentDeleted(
commentId: string,
dumpId: string,
): void {
for (const client of clients) {
send(client.socket, { type: "comment_deleted", commentId, dumpId });
}
broadcast({ type: "comment_deleted", commentId, dumpId });
}
export function broadcastCommentUpdated(comment: Comment): void {
for (const client of clients) {
send(client.socket, { type: "comment_updated", comment });
}
broadcast({ type: "comment_updated", comment });
}
// Global chat room for members: history is behind auth, so the live stream is
// too — an anonymous socket gets nothing chat-related.
function sendToMembers(data: ServerToClientMessage): void {
for (const client of clients) {
if (client.userId) send(client.socket, data);
}
broadcast(data, (client) => !!client.userId);
}
// `nonce` is the sender's id for its pending copy; harmless to everyone else.
@@ -346,6 +326,7 @@ export function handleClientPong(client: WsClient): void {
// Keepalive: ping all clients every 30s, disconnect non-responsive ones
const PING_INTERVAL = 30_000;
const PING = JSON.stringify({ type: "ping" } satisfies ServerToClientMessage);
setInterval(() => {
for (const client of clients) {
@@ -360,6 +341,6 @@ setInterval(() => {
continue;
}
client.pongReceived = false;
send(client.socket, { type: "ping" });
sendRaw(client.socket, PING);
}
}, PING_INTERVAL);

View File

@@ -1,4 +1,5 @@
import { APIErrorCode, APIException } from "../model/interfaces.ts";
import { readTextCapped } from "./rich-content-service.ts";
/**
* Resolves a YouTube playlist to its tracklist, at play time.
@@ -183,7 +184,7 @@ export async function resolveYouTubePlaylist(
`YouTube returned ${res.status}`,
);
}
html = await res.text();
html = await readTextCapped(res);
} catch (err) {
if (err instanceof APIException) throw err;
throw new APIException(