/** * Sends each user their weekly digest and monthly recap, unless they turned * it off. `main.ts` starts {@link startDigestScheduler}, which every quarter * of an hour sends the digests that are due; `digest_schedule` remembers, for * each user and cadence, the start of the period their next one covers. * * A period is 7 days or 30, rolling from when the user's schedule began, so * the digests spread over the week instead of leaving together. A digest * covers the period just ended, or the last full period when the server was * down longer than one; a digest with nothing to show isn't sent, and either * way the next period begins. * * @module */ import { db, transaction } from "../db/connection.ts"; import { FRONTEND_URL, PUBLIC_URL } from "../config.ts"; import { DEFAULT_SETTINGS, DIGEST_CADENCES, DIGEST_SETTING, type DigestCadence, } from "../../shared/settings.ts"; import { SETTINGS_COLUMNS } from "../model/settings.ts"; import type { NotificationType } from "../../shared/notification.ts"; import { type ActivityKind, type DigestContent, type DigestDump, type DigestHighlight, renderDigestEmail, } from "../mail/digest.ts"; import { lookOf } from "../mail/layout.ts"; import { createUnsubscribeToken } from "../lib/jwt.ts"; import { isEmailEnabled, sendEmail } from "./email-service.ts"; import { getSettings } from "./settings-service.ts"; import { dumpFileVersion } from "./dump-service.ts"; const DAY_MS = 86_400_000; /** How long each cadence's period lasts. */ export const PERIOD_MS: Record = { weekly: 7 * DAY_MS, monthly: 30 * DAY_MS, }; const TICK_MS = 15 * 60_000; // What one digest shows at most, as the drafts settled it. const HIGHLIGHTS_MAX = 5; const NETWORK_MAX = 6; const TOP_MAX: Record = { weekly: 5, monthly: 10 }; const MEMBERS_MAX = 10; const PLAYLISTS_MAX = 5; const QUOTE_MAX = 140; const COUNTED: Partial> = { dump_upvoted: "upvotes", user_followed: "followers", dump_commented: "comments", user_mentioned: "mentions", comment_liked: "likes", playlist_followed: "playlistFollows", }; /** A period as the ISO strings dates are stored in, for comparing as text. */ interface Window { from: string; to: string; } const dumpUrl = (row: { id: string; slug: string | null }) => `${FRONTEND_URL}/dumps/${encodeURIComponent(row.slug ?? row.id)}`; const playlistUrl = (row: { id: string; slug: string | null }) => `${FRONTEND_URL}/playlists/${encodeURIComponent(row.slug ?? row.id)}`; /** Flattens Markdown into a short line of text for a quote. */ function quoteOf(markdown: string | undefined | null): string | undefined { if (!markdown) return undefined; const flat = markdown .replace(/!\[[^\]]*\]\([^)]*\)/g, "") .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") .replace(/[`*_>#~]/g, "") .replace(/\s+/g, " ") .trim(); if (!flat) return undefined; return flat.length > QUOTE_MAX ? `${flat.slice(0, QUOTE_MAX - 1).trimEnd()}…` : flat; } /** The columns {@link toDigestDump} reads, selected from `dumps d`. */ const DUMP_COLS = `d.id, d.slug, d.title, d.kind, d.url, d.user_id, d.created_at, d.updated_at, d.file_mime, d.file_size, d.rich_content, d.custom_thumbnail_mime, d.vote_count, u.username AS poster, (SELECT COUNT(*) FROM comments c WHERE c.dump_id = d.id AND c.deleted = 0) AS comment_count`; interface DumpRow { id: string; slug: string | null; title: string; kind: string; url: string | null; user_id: string; created_at: string; updated_at: string | null; file_mime: string | null; file_size: number | null; rich_content: string | null; custom_thumbnail_mime: string | null; vote_count: number; poster: string; comment_count: number; } /** * Returns the image a mail client can load for `row`, which must be public: * the thumbnails route for a custom thumbnail or an image or video upload, * else the link preview's image, through the image proxy when it is plain * http, which a mail client may refuse. */ function thumbnailOf(row: DumpRow): string | null { const isMedia = row.kind === "file" && /^(image|video)\//.test(row.file_mime ?? ""); if (row.custom_thumbnail_mime || isMedia) { const version = dumpFileVersion({ fileSize: row.file_size ?? undefined, createdAt: new Date(row.created_at), updatedAt: row.updated_at ? new Date(row.updated_at) : undefined, }); return `${PUBLIC_URL}/api/thumbnails/${row.id}?v=${version}`; } let thumbnailUrl: unknown; try { thumbnailUrl = row.rich_content ? JSON.parse(row.rich_content)?.thumbnailUrl : undefined; } catch { return null; } if (typeof thumbnailUrl !== "string") return null; if (thumbnailUrl.startsWith("https://")) return thumbnailUrl; if (thumbnailUrl.startsWith("http://")) { return `${PUBLIC_URL}/api/proxy-image?url=${ encodeURIComponent(thumbnailUrl) }`; } return null; } function toDigestDump(row: DumpRow, readerId: string): DigestDump { let domain: string | null = null; if (row.kind === "url" && row.url) { try { domain = new URL(row.url).hostname.replace(/^www\./, ""); } catch { domain = null; } } return { title: row.title, url: dumpUrl(row), poster: row.poster, domain, thumbnail: thumbnailOf(row), votes: row.vote_count, comments: row.comment_count, mine: row.user_id === readerId, }; } /** * Returns what happened to the reader: their notifications of the period, * counted by kind, and the few worth reading, mentions and comments first. */ function activityOf( readerId: string, w: Window, ): DigestContent["activity"] { const byType = db.prepare( `SELECT type, COUNT(*) AS n FROM notifications WHERE user_id = ? AND created_at >= ? AND created_at < ? GROUP BY type;`, ).all(readerId, w.from, w.to) as { type: NotificationType; n: number }[]; const counts: Partial> = {}; let total = 0; for (const { type, n } of byType) { total += n; const kind = COUNTED[type]; if (kind) counts[kind] = n; } const rows = db.prepare( `SELECT type, data, created_at FROM notifications WHERE user_id = ? AND created_at >= ? AND created_at < ? AND type IN ('user_mentioned', 'dump_commented', 'playlist_followed', 'user_followed', 'comment_liked') ORDER BY CASE type WHEN 'user_mentioned' THEN 0 WHEN 'dump_commented' THEN 0 WHEN 'comment_liked' THEN 2 ELSE 1 END, created_at DESC LIMIT ?;`, ).all(readerId, w.from, w.to, HIGHLIGHTS_MAX) as { type: NotificationType; data: string; created_at: string; }[]; const highlights = rows.flatMap((row) => { try { const h = highlightOf(row.type, JSON.parse(row.data)); return h ? [{ ...h, at: new Date(row.created_at) }] : []; } catch { return []; } }); if (Object.keys(counts).length === 0 && highlights.length === 0) return null; return { counts, highlights, total }; } const commentBody = (id: string) => (db.prepare(`SELECT body FROM comments WHERE id = ? AND deleted = 0;`) .get(id) as { body: string } | undefined)?.body; const dumpTarget = (id: string, fallbackTitle: string) => { const row = db.prepare(`SELECT id, slug, title FROM dumps WHERE id = ?;`) .get(id) as { id: string; slug: string | null; title: string } | undefined; return row ? { title: row.title, url: dumpUrl(row) } : { title: fallbackTitle, url: `${FRONTEND_URL}/dumps/${encodeURIComponent(id)}`, }; }; const playlistTarget = (id: string, fallbackTitle: string) => { const row = db.prepare(`SELECT id, slug, title FROM playlists WHERE id = ?;`) .get(id) as { id: string; slug: string | null; title: string } | undefined; return row ? { title: row.title, url: playlistUrl(row) } : { title: fallbackTitle, url: `${FRONTEND_URL}/playlists/${encodeURIComponent(id)}`, }; }; /** Returns the highlight a notification makes, or null for another kind. */ function highlightOf( type: NotificationType, // deno-lint-ignore no-explicit-any data: any, ): Omit | null { switch (type) { case "user_mentioned": switch (data.contextType) { case "comment": return { kind: "mentionComment", who: data.mentionerUsername, target: dumpTarget(data.dumpId, data.contextTitle), quote: quoteOf(commentBody(data.contextId)), }; case "dump": return { kind: "mentionDump", who: data.mentionerUsername, target: dumpTarget(data.contextId, data.contextTitle), }; case "playlist": return { kind: "mentionPlaylist", who: data.mentionerUsername, target: playlistTarget(data.contextId, data.contextTitle), }; case "chat": return { kind: "mentionChat", who: data.mentionerUsername, quote: quoteOf( (db.prepare(`SELECT body FROM chat_messages WHERE id = ?;`) .get(data.contextId) as { body: string } | undefined)?.body, ), }; } return null; case "dump_commented": return { kind: "comment", who: data.commenterUsername, target: dumpTarget(data.dumpId, data.dumpTitle), quote: quoteOf(commentBody(data.commentId)), }; case "playlist_followed": return { kind: "playlistFollow", who: data.followerUsername, target: playlistTarget(data.playlistId, data.playlistTitle), }; case "user_followed": return { kind: "userFollow", who: data.followerUsername }; case "comment_liked": return { kind: "commentLike", who: data.likerUsername, target: dumpTarget(data.dumpId, data.dumpTitle), quote: quoteOf(commentBody(data.commentId)), }; } return null; } /** * Returns the public dumps of the period from the users and playlists the * reader follows, the most upvoted first, without their own. */ function networkOf(readerId: string, w: Window): DigestContent["network"] { const rows = db.prepare( `SELECT ${DUMP_COLS}, NULL AS playlist_title, d.created_at AS at FROM dumps d JOIN users u ON u.id = d.user_id WHERE d.is_private = 0 AND d.user_id != :reader AND d.created_at >= :from AND d.created_at < :to AND d.user_id IN (SELECT followed_user_id FROM follows WHERE follower_id = :reader AND followed_user_id IS NOT NULL) UNION ALL SELECT ${DUMP_COLS}, p.title AS playlist_title, pd.added_at AS at FROM playlist_dumps pd JOIN playlists p ON p.id = pd.playlist_id JOIN dumps d ON d.id = pd.dump_id JOIN users u ON u.id = d.user_id WHERE p.is_public = 1 AND d.is_private = 0 AND d.user_id != :reader AND pd.added_at >= :from AND pd.added_at < :to AND pd.playlist_id IN (SELECT followed_playlist_id FROM follows WHERE follower_id = :reader AND followed_playlist_id IS NOT NULL);`, ).all({ reader: readerId, from: w.from, to: w.to }) as unknown as (DumpRow & { playlist_title: string | null; at: string; })[]; // A dump both posted by someone followed and added to a followed playlist // is listed once, as the post. const byId = new Map(); for (const row of rows) { const seen = byId.get(row.id); if (!seen || (seen.playlist_title && !row.playlist_title)) { byId.set(row.id, row); } } const sorted = [...byId.values()].sort((a, b) => b.vote_count - a.vote_count || b.at.localeCompare(a.at) ); if (sorted.length === 0) return null; return { dumps: sorted.slice(0, NETWORK_MAX).map((row) => ({ ...toDigestDump(row, readerId), ...(row.playlist_title ? { addedTo: row.playlist_title } : {}), })), more: Math.max(0, sorted.length - NETWORK_MAX), }; } /** * Returns the period's most upvoted public dumps among those posted in it, * the newer first on a tie, as the "hot" tab would. */ function topOf( readerId: string, cadence: DigestCadence, w: Window, ): DigestDump[] { const rows = db.prepare( `SELECT ${DUMP_COLS} FROM dumps d JOIN users u ON u.id = d.user_id WHERE d.is_private = 0 AND d.vote_count > 0 AND d.created_at >= ? AND d.created_at < ? ORDER BY d.vote_count DESC, d.created_at DESC LIMIT ?;`, ).all(w.from, w.to, TOP_MAX[cadence]) as unknown as DumpRow[]; return rows.map((row) => toDigestDump(row, readerId)); } const countOf = (sql: string, ...params: string[]) => (db.prepare(sql).get(...params) as { n: number }).n; /** * Returns what the whole site did in the period: who joined, the new public * playlists, the chat's activity and the totals. */ function communityOf(readerId: string, w: Window): DigestContent["community"] { const members = db.prepare( `SELECT u.username, i.username AS invited_by FROM users u LEFT JOIN users i ON i.id = u.invited_by WHERE u.created_at >= ? AND u.created_at < ? AND u.id != ? ORDER BY u.created_at;`, ).all(w.from, w.to, readerId) as { username: string; invited_by: string | null; }[]; const playlists = db.prepare( `SELECT p.id, p.slug, p.title, u.username AS owner, (SELECT COUNT(*) FROM playlist_dumps pd WHERE pd.playlist_id = p.id) AS dumps FROM playlists p JOIN users u ON u.id = p.user_id WHERE p.is_public = 1 AND p.created_at >= ? AND p.created_at < ? ORDER BY dumps DESC, p.created_at DESC;`, ).all(w.from, w.to) as { id: string; slug: string | null; title: string; owner: string; dumps: number; }[]; const chatMessages = countOf( `SELECT COUNT(*) AS n FROM chat_messages WHERE created_at >= ? AND created_at < ?;`, w.from, w.to, ); const totals = { dumps: countOf( `SELECT COUNT(*) AS n FROM dumps WHERE is_private = 0 AND created_at >= ? AND created_at < ?;`, w.from, w.to, ), upvotes: countOf( `SELECT COUNT(*) AS n FROM votes WHERE created_at >= ? AND created_at < ?;`, w.from, w.to, ), comments: countOf( `SELECT COUNT(*) AS n FROM comments WHERE deleted = 0 AND created_at >= ? AND created_at < ?;`, w.from, w.to, ), }; if ( members.length + playlists.length + chatMessages + totals.dumps + totals.upvotes + totals.comments === 0 ) return null; return { members: members.slice(0, MEMBERS_MAX).map((m) => ({ username: m.username, invitedBy: m.invited_by, })), membersMore: Math.max(0, members.length - MEMBERS_MAX), playlists: playlists.slice(0, PLAYLISTS_MAX).map((p) => ({ title: p.title, url: playlistUrl(p), owner: p.owner, dumps: p.dumps, })), playlistsMore: Math.max(0, playlists.length - PLAYLISTS_MAX), chatMessages, totals, }; } /** * Returns the digest of user `readerId` for the period from `from` to `to`, * or null when it would show nothing. The links are absolute; the * unsubscribe one carries `token`, which turns off `cadence`. */ export function buildDigest( readerId: string, reader: string, cadence: DigestCadence, from: Date, to: Date, token: string, ): DigestContent | null { const w = { from: from.toISOString(), to: to.toISOString() }; const weekly = cadence === "weekly"; const activity = weekly ? activityOf(readerId, w) : null; const network = weekly ? networkOf(readerId, w) : null; const top = topOf(readerId, cadence, w); const community = communityOf(readerId, w); if (!activity && !network && top.length === 0 && !community) return null; return { cadence, from, to, reader, activity, network, top, community, links: { notifications: `${FRONTEND_URL}/notifications`, followed: `${FRONTEND_URL}/~/followed`, chat: `${FRONTEND_URL}/?chat=open`, settings: `${FRONTEND_URL}/users/${ encodeURIComponent(reader) }/~/settings`, unsubscribe: `${FRONTEND_URL}/unsubscribe?token=${ encodeURIComponent(token) }`, }, }; } /** * Brings `digest_schedule` in line with the settings: a user who turned a * cadence off loses its row, and one who receives it without a row, being * new or having turned it back on, starts a period at `now`. */ export function syncDigestSchedules(now: Date): void { transaction(() => { for (const cadence of DIGEST_CADENCES) { const setting = DIGEST_SETTING[cadence]; const column = SETTINGS_COLUMNS[setting]; const byDefault = DEFAULT_SETTINGS[setting] ? 1 : 0; db.prepare( `DELETE FROM digest_schedule WHERE cadence = ? AND user_id IN (SELECT user_id FROM user_settings WHERE ${column} = 0);`, ).run(cadence); db.prepare( `INSERT INTO digest_schedule (user_id, cadence, window_start) SELECT u.id, ?, ? FROM users u LEFT JOIN user_settings s ON s.user_id = u.id WHERE COALESCE(s.${column}, ${byDefault}) = 1 AND NOT EXISTS (SELECT 1 FROM digest_schedule ds WHERE ds.user_id = u.id AND ds.cadence = ?);`, ).run(cadence, now.toISOString(), cadence); } }); } /** * Sends every digest due at `now`, one after the other, and returns how many * went out. A digest that fails to send is logged and not retried: its * period is over either way, and the next one starts. */ export async function sendDueDigests(now: Date = new Date()): Promise { if (!isEmailEnabled()) return 0; syncDigestSchedules(now); let sent = 0; for (const cadence of DIGEST_CADENCES) { const period = PERIOD_MS[cadence]; const due = db.prepare( `SELECT ds.user_id, ds.window_start, u.username, u.email FROM digest_schedule ds JOIN users u ON u.id = ds.user_id WHERE ds.cadence = ? AND ds.window_start <= ?;`, ).all(cadence, new Date(now.getTime() - period).toISOString()) as { user_id: string; window_start: string; username: string; email: string; }[]; for (const row of due) { let from = new Date(row.window_start); let to = new Date(from.getTime() + period); // More than a period late (the server was down): cover the last full // period rather than one long gone. if (to.getTime() + period <= now.getTime()) { from = new Date(now.getTime() - period); to = now; } db.prepare( `UPDATE digest_schedule SET window_start = ? WHERE user_id = ? AND cadence = ?;`, ).run(to.toISOString(), row.user_id, cadence); // An address with no dot in its domain, such as the first admin's // admin@localhost, can't receive mail. if (!/@[^@]+\.[^@]+$/.test(row.email)) continue; const token = await createUnsubscribeToken(row.user_id, cadence); const content = buildDigest( row.user_id, row.username, cadence, from, to, token, ); if (!content) continue; const email = renderDigestEmail( content, lookOf(getSettings(row.user_id)), ); try { await sendEmail({ to: row.email, ...email, headers: { "List-Unsubscribe": `<${PUBLIC_URL}/api/users/unsubscribe?token=${ encodeURIComponent(token) }>`, "List-Unsubscribe-Post": "List-Unsubscribe=One-Click", }, }); sent++; } catch (err) { console.error(`[digest] ${cadence} to ${row.username} failed:`, err); } } } return sent; } /** * Sends the due digests a minute after the server starts, then every quarter * of an hour. A run still sending when the next is due makes that one skip. * `main.ts` calls it when run directly, never in the tests. */ export function startDigestScheduler(): void { let running = false; const tick = () => { if (running) return; running = true; sendDueDigests() .then((n) => n > 0 && console.log(`[digest] sent ${n}`)) .catch((err) => console.error("[digest] run failed:", err)) .finally(() => running = false); }; setTimeout(tick, 60_000); setInterval(tick, TICK_MS); }