Files
gerbeur/api/services/digest-service.ts
khannurien 4468438ba7 v4: every page and API URL follows one of two rules, written down in API.md — pages are a hierarchy where @name is a user and any other root word a category or an app page, tabs and views are plain segments and a default view is the bare path; API resources are /api/<plural>/<key>, a user's key being an id or @name, the caller's things under /api/me, media a sub-resource, lookups by another key a query filter. ~ is gone: the feed is /, /hot, /new, /journal, /follow and /music/hot, a profile /@bob with its overview and the tabs dumps, upvoted, playlists (her own only), follow (users, and /follow/playlists), invitees and manage (role and name, for user:manage), which replace the separate UserDumps, UserUpvoted and UserPlaylists pages; /settings holds the account (now with the username) and /manage the category manager, both linked from the user menu and one's profile. Dumps and playlists live under their owner, /@bob/dumps/<slug> and /@bob/playlists/<slug>, named by slug alone: migration 0021 re-slugs them to the bare slugify(title), unique per owner and kind (the oldest keeps it, clashes get -2), renames the categories whose slug v4 reserves (hot, follow, settings…), and rewrites this site's v3 links in dump notes, comments, chat, and playlist and profile descriptions. A slug changes only when edited, through the edit forms' new link field, which proposes slugify(title) as the title changes; a username changes from /settings or, for user:manage, from /@name/manage. Every slug or name left behind stays an alias of its item or user for good (dump_slug_aliases, playlist_slug_aliases, user_aliases), so old links redirect without chaining and nobody else can take one; slugs change 3 times a day per item, names once in 30 days, moderators exempt. An alias, or a name in another case, answers the item with its current path: the pages put it in their URL (useCanonicalPath, without fetching again), and the OG middleware answers a 301. shared/paths.ts builds and parses every page path, for the client's links, the emails' (digest-service lost its own builders and its id fallbacks, and links /follow and /settings) and the server's matchers (OG tags, self previews, backlinks, self-url); shared/slug.ts holds slugify and the format categories share. Dumps and playlists carry ownerUsername and a required slug, and notifications join their users' current names and their items' paths each time they are sent or read, so renames reach them and a deleted item's has no link. The API was renamed per the plan's table: POST /api/users and /api/sessions, /api/password-resets, /api/me (with settings, password, avatar, media-token, playlists?contains=, follow, feed?from=, notifications), /api/users/:key/{dumps,upvoted,playlists,follow/users,follow/playlists,invitees,avatar} and by path /api/users/:key/{dumps,playlists}/:slug, /api/dumps?url= and ?category=, /api/dumps/:id/{file,thumbnail,metadata}, /api/playlists/:id/cover, /api/remote/{preview,image,bandcamp,youtube} (one router), and /api/unsubscribe, the v3 POST /api/users/unsubscribe staying for the digests already sent. A stored default feed tab "followed" reads as "follow" once. dumps.slug and playlists.slug stay nullable, since SQLite can't add NOT NULL without rebuilding tables others reference; every writer sets them and unique (user_id, slug) indexes hold them. 254 backend tests (new: migration 0021 on a v3 fixture, slug aliases, swaps and rate limits, username renames, aliases and limits, reads by path and their privacy, the OG 301s, both unsubscribe paths, v4-shaped digest links with no ids, notification joins, shared paths and slugs) and 668 frontend tests (new: the profile's tabs and follow views, Settings, Manage, the link field, canonical URLs on the dump, playlist and profile pages, bare default feed paths) pass, and tsc, eslint, deno check, deno lint, deno fmt and doc:lint are clean; the migration was also run on a copy of the development database.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 16:08:59 +00:00

617 lines
20 KiB
TypeScript

/**
* Sends each user their weekly digest and monthly recap, unless they turned
* it off. `main.ts` starts {@link startDigestScheduler}, which every quarter
* of an hour sends the digests that are due; `digest_schedule` remembers, for
* each user and cadence, the start of the period their next one covers.
*
* A period is 7 days or 30, rolling from when the user's schedule began, so
* the digests spread over the week instead of leaving together. A digest
* covers the period just ended, or the last full period when the server was
* down longer than one; a digest with nothing to show isn't sent, and either
* way the next period begins.
*
* @module
*/
import { db, transaction } from "../db/connection.ts";
import { FRONTEND_URL, PUBLIC_URL } from "../config.ts";
import {
DEFAULT_SETTINGS,
DIGEST_CADENCES,
DIGEST_SETTING,
type DigestCadence,
} from "../../shared/settings.ts";
import { SETTINGS_COLUMNS } from "../model/settings.ts";
import type {
DumpRef,
NotificationType,
PlaylistRef,
} from "../../shared/notification.ts";
import { dumpPath, feedPath, playlistPath } from "../../shared/paths.ts";
import {
type ActivityKind,
type DigestContent,
type DigestDump,
type DigestHighlight,
renderDigestEmail,
} from "../mail/digest.ts";
import { lookOf } from "../mail/layout.ts";
import { createUnsubscribeToken } from "../lib/jwt.ts";
import { isEmailEnabled, sendEmail } from "./email-service.ts";
import { getSettings } from "./settings-service.ts";
import { dumpFileVersion } from "./dump-service.ts";
import { withCurrentRefs } from "./notification-service.ts";
const DAY_MS = 86_400_000;
/** How long each cadence's period lasts. */
export const PERIOD_MS: Record<DigestCadence, number> = {
weekly: 7 * DAY_MS,
monthly: 30 * DAY_MS,
};
const TICK_MS = 15 * 60_000;
// What one digest shows at most, as the drafts settled it.
const HIGHLIGHTS_MAX = 5;
const NETWORK_MAX = 6;
const TOP_MAX: Record<DigestCadence, number> = { weekly: 5, monthly: 10 };
const MEMBERS_MAX = 10;
const PLAYLISTS_MAX = 5;
const QUOTE_MAX = 140;
const COUNTED: Partial<Record<NotificationType, ActivityKind>> = {
dump_upvoted: "upvotes",
user_followed: "followers",
dump_commented: "comments",
user_mentioned: "mentions",
comment_liked: "likes",
playlist_followed: "playlistFollows",
};
/** A period as the ISO strings dates are stored in, for comparing as text. */
interface Window {
from: string;
to: string;
}
/** Flattens Markdown into a short line of text for a quote. */
function quoteOf(markdown: string | undefined | null): string | undefined {
if (!markdown) return undefined;
const flat = markdown
.replace(/!\[[^\]]*\]\([^)]*\)/g, "")
.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
.replace(/[`*_>#~]/g, "")
.replace(/\s+/g, " ")
.trim();
if (!flat) return undefined;
return flat.length > QUOTE_MAX
? `${flat.slice(0, QUOTE_MAX - 1).trimEnd()}…`
: flat;
}
/** The columns {@link toDigestDump} reads, selected from `dumps d`. */
const DUMP_COLS = `d.id, d.slug, d.title, d.kind, d.url, d.user_id,
d.created_at, d.updated_at, d.file_mime, d.file_size, d.rich_content,
d.custom_thumbnail_mime, d.vote_count, u.username AS poster,
(SELECT COUNT(*) FROM comments c WHERE c.dump_id = d.id AND c.deleted = 0) AS comment_count`;
interface DumpRow {
id: string;
slug: string;
title: string;
kind: string;
url: string | null;
user_id: string;
created_at: string;
updated_at: string | null;
file_mime: string | null;
file_size: number | null;
rich_content: string | null;
custom_thumbnail_mime: string | null;
vote_count: number;
poster: string;
comment_count: number;
}
/**
* Returns the image a mail client can load for `row`, which must be public:
* the thumbnails route for a custom thumbnail or an image or video upload,
* else the link preview's image, through the image proxy when it is plain
* http, which a mail client may refuse.
*/
function thumbnailOf(row: DumpRow): string | null {
const isMedia = row.kind === "file" &&
/^(image|video)\//.test(row.file_mime ?? "");
if (row.custom_thumbnail_mime || isMedia) {
const version = dumpFileVersion({
fileSize: row.file_size ?? undefined,
createdAt: new Date(row.created_at),
updatedAt: row.updated_at ? new Date(row.updated_at) : undefined,
});
return `${PUBLIC_URL}/api/dumps/${row.id}/thumbnail?v=${version}`;
}
let thumbnailUrl: unknown;
try {
thumbnailUrl = row.rich_content
? JSON.parse(row.rich_content)?.thumbnailUrl
: undefined;
} catch {
return null;
}
if (typeof thumbnailUrl !== "string") return null;
if (thumbnailUrl.startsWith("https://")) return thumbnailUrl;
if (thumbnailUrl.startsWith("http://")) {
return `${PUBLIC_URL}/api/remote/image?url=${
encodeURIComponent(thumbnailUrl)
}`;
}
return null;
}
function toDigestDump(row: DumpRow, readerId: string): DigestDump {
let domain: string | null = null;
if (row.kind === "url" && row.url) {
try {
domain = new URL(row.url).hostname.replace(/^www\./, "");
} catch {
domain = null;
}
}
return {
title: row.title,
url: `${FRONTEND_URL}${
dumpPath({ ownerUsername: row.poster, slug: row.slug })
}`,
poster: row.poster,
domain,
thumbnail: thumbnailOf(row),
votes: row.vote_count,
comments: row.comment_count,
mine: row.user_id === readerId,
};
}
/**
* Returns what happened to the reader: their notifications of the period,
* counted by kind, and the few worth reading, mentions and comments first.
*/
function activityOf(
readerId: string,
w: Window,
): DigestContent["activity"] {
const byType = db.prepare(
`SELECT type, COUNT(*) AS n FROM notifications
WHERE user_id = ? AND created_at >= ? AND created_at < ?
GROUP BY type;`,
).all(readerId, w.from, w.to) as { type: NotificationType; n: number }[];
const counts: Partial<Record<ActivityKind, number>> = {};
let total = 0;
for (const { type, n } of byType) {
total += n;
const kind = COUNTED[type];
if (kind) counts[kind] = n;
}
const rows = db.prepare(
`SELECT type, data, created_at FROM notifications
WHERE user_id = ? AND created_at >= ? AND created_at < ?
AND type IN ('user_mentioned', 'dump_commented', 'playlist_followed',
'user_followed', 'comment_liked')
ORDER BY CASE type
WHEN 'user_mentioned' THEN 0 WHEN 'dump_commented' THEN 0
WHEN 'comment_liked' THEN 2 ELSE 1 END,
created_at DESC
LIMIT ?;`,
).all(readerId, w.from, w.to, HIGHLIGHTS_MAX) as {
type: NotificationType;
data: string;
created_at: string;
}[];
const highlights = rows.flatMap((row) => {
try {
const h = highlightOf(row.type, withCurrentRefs(JSON.parse(row.data)));
return h ? [{ ...h, at: new Date(row.created_at) }] : [];
} catch {
return [];
}
});
if (Object.keys(counts).length === 0 && highlights.length === 0) return null;
return { counts, highlights, total };
}
const commentBody = (id: string) =>
(db.prepare(`SELECT body FROM comments WHERE id = ? AND deleted = 0;`)
.get(id) as { body: string } | undefined)?.body;
/**
* Returns a highlight's dump, titled `title`, linked where the notification's
* joined ref says it is now, or unlinked once it is deleted.
*/
const dumpTarget = (ref: DumpRef, title: string) => ({
title,
url: ref.dumpSlug && ref.dumpOwnerUsername
? `${FRONTEND_URL}${
dumpPath({ ownerUsername: ref.dumpOwnerUsername, slug: ref.dumpSlug })
}`
: null,
});
/** Returns a highlight's playlist as {@link dumpTarget} does its dump. */
const playlistTarget = (ref: PlaylistRef, title: string) => ({
title,
url: ref.playlistSlug && ref.playlistOwnerUsername
? `${FRONTEND_URL}${
playlistPath({
ownerUsername: ref.playlistOwnerUsername,
slug: ref.playlistSlug,
})
}`
: null,
});
/**
* Returns the highlight a notification makes, from its data with the refs
* joined, or null for another kind.
*/
function highlightOf(
type: NotificationType,
// deno-lint-ignore no-explicit-any
data: any,
): Omit<DigestHighlight, "at"> | null {
switch (type) {
case "user_mentioned":
switch (data.contextType) {
case "comment":
return {
kind: "mentionComment",
who: data.mentionerUsername,
target: dumpTarget(data, data.contextTitle),
quote: quoteOf(commentBody(data.contextId)),
};
case "dump":
return {
kind: "mentionDump",
who: data.mentionerUsername,
target: dumpTarget(data, data.contextTitle),
};
case "playlist":
return {
kind: "mentionPlaylist",
who: data.mentionerUsername,
target: playlistTarget(data, data.contextTitle),
};
case "chat":
return {
kind: "mentionChat",
who: data.mentionerUsername,
quote: quoteOf(
(db.prepare(`SELECT body FROM chat_messages WHERE id = ?;`)
.get(data.contextId) as { body: string } | undefined)?.body,
),
};
}
return null;
case "dump_commented":
return {
kind: "comment",
who: data.commenterUsername,
target: dumpTarget(data, data.dumpTitle),
quote: quoteOf(commentBody(data.commentId)),
};
case "playlist_followed":
return {
kind: "playlistFollow",
who: data.followerUsername,
target: playlistTarget(data, data.playlistTitle),
};
case "user_followed":
return { kind: "userFollow", who: data.followerUsername };
case "comment_liked":
return {
kind: "commentLike",
who: data.likerUsername,
target: dumpTarget(data, data.dumpTitle),
quote: quoteOf(commentBody(data.commentId)),
};
}
return null;
}
/**
* Returns the public dumps of the period from the users and playlists the
* reader follows, the most upvoted first, without their own.
*/
function networkOf(readerId: string, w: Window): DigestContent["network"] {
const rows = db.prepare(
`SELECT ${DUMP_COLS}, NULL AS playlist_title, d.created_at AS at
FROM dumps d JOIN users u ON u.id = d.user_id
WHERE d.is_private = 0 AND d.user_id != :reader
AND d.created_at >= :from AND d.created_at < :to
AND d.user_id IN (SELECT followed_user_id FROM follows
WHERE follower_id = :reader AND followed_user_id IS NOT NULL)
UNION ALL
SELECT ${DUMP_COLS}, p.title AS playlist_title, pd.added_at AS at
FROM playlist_dumps pd
JOIN playlists p ON p.id = pd.playlist_id
JOIN dumps d ON d.id = pd.dump_id
JOIN users u ON u.id = d.user_id
WHERE p.is_public = 1 AND d.is_private = 0 AND d.user_id != :reader
AND pd.added_at >= :from AND pd.added_at < :to
AND pd.playlist_id IN (SELECT followed_playlist_id FROM follows
WHERE follower_id = :reader AND followed_playlist_id IS NOT NULL);`,
).all({ reader: readerId, from: w.from, to: w.to }) as unknown as (DumpRow & {
playlist_title: string | null;
at: string;
})[];
// A dump both posted by someone followed and added to a followed playlist
// is listed once, as the post.
const byId = new Map<string, (typeof rows)[number]>();
for (const row of rows) {
const seen = byId.get(row.id);
if (!seen || (seen.playlist_title && !row.playlist_title)) {
byId.set(row.id, row);
}
}
const sorted = [...byId.values()].sort((a, b) =>
b.vote_count - a.vote_count || b.at.localeCompare(a.at)
);
if (sorted.length === 0) return null;
return {
dumps: sorted.slice(0, NETWORK_MAX).map((row) => ({
...toDigestDump(row, readerId),
...(row.playlist_title ? { addedTo: row.playlist_title } : {}),
})),
more: Math.max(0, sorted.length - NETWORK_MAX),
};
}
/**
* Returns the period's most upvoted public dumps among those posted in it,
* the newer first on a tie, as the "hot" tab would.
*/
function topOf(
readerId: string,
cadence: DigestCadence,
w: Window,
): DigestDump[] {
const rows = db.prepare(
`SELECT ${DUMP_COLS} FROM dumps d JOIN users u ON u.id = d.user_id
WHERE d.is_private = 0 AND d.vote_count > 0
AND d.created_at >= ? AND d.created_at < ?
ORDER BY d.vote_count DESC, d.created_at DESC
LIMIT ?;`,
).all(w.from, w.to, TOP_MAX[cadence]) as unknown as DumpRow[];
return rows.map((row) => toDigestDump(row, readerId));
}
const countOf = (sql: string, ...params: string[]) =>
(db.prepare(sql).get(...params) as { n: number }).n;
/**
* Returns what the whole site did in the period: who joined, the new public
* playlists, the chat's activity and the totals.
*/
function communityOf(readerId: string, w: Window): DigestContent["community"] {
const members = db.prepare(
`SELECT u.username, i.username AS invited_by
FROM users u LEFT JOIN users i ON i.id = u.invited_by
WHERE u.created_at >= ? AND u.created_at < ? AND u.id != ?
ORDER BY u.created_at;`,
).all(w.from, w.to, readerId) as {
username: string;
invited_by: string | null;
}[];
const playlists = db.prepare(
`SELECT p.id, p.slug, p.title, u.username AS owner,
(SELECT COUNT(*) FROM playlist_dumps pd WHERE pd.playlist_id = p.id) AS dumps
FROM playlists p JOIN users u ON u.id = p.user_id
WHERE p.is_public = 1 AND p.created_at >= ? AND p.created_at < ?
ORDER BY dumps DESC, p.created_at DESC;`,
).all(w.from, w.to) as {
id: string;
slug: string;
title: string;
owner: string;
dumps: number;
}[];
const chatMessages = countOf(
`SELECT COUNT(*) AS n FROM chat_messages WHERE created_at >= ? AND created_at < ?;`,
w.from,
w.to,
);
const totals = {
dumps: countOf(
`SELECT COUNT(*) AS n FROM dumps WHERE is_private = 0 AND created_at >= ? AND created_at < ?;`,
w.from,
w.to,
),
upvotes: countOf(
`SELECT COUNT(*) AS n FROM votes WHERE created_at >= ? AND created_at < ?;`,
w.from,
w.to,
),
comments: countOf(
`SELECT COUNT(*) AS n FROM comments WHERE deleted = 0 AND created_at >= ? AND created_at < ?;`,
w.from,
w.to,
),
};
if (
members.length + playlists.length + chatMessages + totals.dumps +
totals.upvotes + totals.comments === 0
) return null;
return {
members: members.slice(0, MEMBERS_MAX).map((m) => ({
username: m.username,
invitedBy: m.invited_by,
})),
membersMore: Math.max(0, members.length - MEMBERS_MAX),
playlists: playlists.slice(0, PLAYLISTS_MAX).map((p) => ({
title: p.title,
url: `${FRONTEND_URL}${
playlistPath({ ownerUsername: p.owner, slug: p.slug })
}`,
owner: p.owner,
dumps: p.dumps,
})),
playlistsMore: Math.max(0, playlists.length - PLAYLISTS_MAX),
chatMessages,
totals,
};
}
/**
* Returns the digest of user `readerId` for the period from `from` to `to`,
* or null when it would show nothing. The links are absolute; the
* unsubscribe one carries `token`, which turns off `cadence`.
*/
export function buildDigest(
readerId: string,
reader: string,
cadence: DigestCadence,
from: Date,
to: Date,
token: string,
): DigestContent | null {
const w = { from: from.toISOString(), to: to.toISOString() };
const weekly = cadence === "weekly";
const activity = weekly ? activityOf(readerId, w) : null;
const network = weekly ? networkOf(readerId, w) : null;
const top = topOf(readerId, cadence, w);
const community = communityOf(readerId, w);
if (!activity && !network && top.length === 0 && !community) return null;
return {
cadence,
from,
to,
reader,
activity,
network,
top,
community,
links: {
notifications: `${FRONTEND_URL}/notifications`,
followed: `${FRONTEND_URL}${feedPath(undefined, "follow")}`,
chat: `${FRONTEND_URL}/?chat=open`,
settings: `${FRONTEND_URL}/settings`,
unsubscribe: `${FRONTEND_URL}/unsubscribe?token=${
encodeURIComponent(token)
}`,
},
};
}
/**
* Brings `digest_schedule` in line with the settings: a user who turned a
* cadence off loses its row, and one who receives it without a row, being
* new or having turned it back on, starts a period at `now`.
*/
export function syncDigestSchedules(now: Date): void {
transaction(() => {
for (const cadence of DIGEST_CADENCES) {
const setting = DIGEST_SETTING[cadence];
const column = SETTINGS_COLUMNS[setting];
const byDefault = DEFAULT_SETTINGS[setting] ? 1 : 0;
db.prepare(
`DELETE FROM digest_schedule WHERE cadence = ? AND user_id IN
(SELECT user_id FROM user_settings WHERE ${column} = 0);`,
).run(cadence);
db.prepare(
`INSERT INTO digest_schedule (user_id, cadence, window_start)
SELECT u.id, ?, ? FROM users u
LEFT JOIN user_settings s ON s.user_id = u.id
WHERE COALESCE(s.${column}, ${byDefault}) = 1
AND NOT EXISTS (SELECT 1 FROM digest_schedule ds
WHERE ds.user_id = u.id AND ds.cadence = ?);`,
).run(cadence, now.toISOString(), cadence);
}
});
}
/**
* Sends every digest due at `now`, one after the other, and returns how many
* went out. A digest that fails to send is logged and not retried: its
* period is over either way, and the next one starts.
*/
export async function sendDueDigests(now: Date = new Date()): Promise<number> {
if (!isEmailEnabled()) return 0;
syncDigestSchedules(now);
let sent = 0;
for (const cadence of DIGEST_CADENCES) {
const period = PERIOD_MS[cadence];
const due = db.prepare(
`SELECT ds.user_id, ds.window_start, u.username, u.email
FROM digest_schedule ds JOIN users u ON u.id = ds.user_id
WHERE ds.cadence = ? AND ds.window_start <= ?;`,
).all(cadence, new Date(now.getTime() - period).toISOString()) as {
user_id: string;
window_start: string;
username: string;
email: string;
}[];
for (const row of due) {
let from = new Date(row.window_start);
let to = new Date(from.getTime() + period);
// More than a period late (the server was down): cover the last full
// period rather than one long gone.
if (to.getTime() + period <= now.getTime()) {
from = new Date(now.getTime() - period);
to = now;
}
db.prepare(
`UPDATE digest_schedule SET window_start = ? WHERE user_id = ? AND cadence = ?;`,
).run(to.toISOString(), row.user_id, cadence);
// An address with no dot in its domain, such as the first admin's
// admin@localhost, can't receive mail.
if (!/@[^@]+\.[^@]+$/.test(row.email)) continue;
const token = await createUnsubscribeToken(row.user_id, cadence);
const content = buildDigest(
row.user_id,
row.username,
cadence,
from,
to,
token,
);
if (!content) continue;
const email = renderDigestEmail(
content,
lookOf(getSettings(row.user_id)),
);
try {
await sendEmail({
to: row.email,
...email,
headers: {
"List-Unsubscribe": `<${PUBLIC_URL}/api/unsubscribe?token=${
encodeURIComponent(token)
}>`,
"List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
},
});
sent++;
} catch (err) {
console.error(`[digest] ${cadence} to ${row.username} failed:`, err);
}
}
}
return sent;
}
/**
* Sends the due digests a minute after the server starts, then every quarter
* of an hour. A run still sending when the next is due makes that one skip.
* `main.ts` calls it when run directly, never in the tests.
*/
export function startDigestScheduler(): void {
let running = false;
const tick = () => {
if (running) return;
running = true;
sendDueDigests()
.then((n) => n > 0 && console.log(`[digest] sent ${n}`))
.catch((err) => console.error("[digest] run failed:", err))
.finally(() => running = false);
};
setTimeout(tick, 60_000);
setInterval(tick, TICK_MS);
}