/** * Sends each user their weekly digest and monthly recap, unless they turned * it off. `main.ts` starts {@link digestJob}, which every quarter of an hour * sends the digests that are due; `digest_schedule` remembers, for * each user and cadence, the start of the period their next one covers. * * A period is 7 days or 30, rolling from when the user's schedule began, so * the digests spread over the week instead of leaving together. A digest * covers the period just ended, or the last full period when the server was * down longer than one; a digest with nothing to show isn't sent, and either * way the next period begins. The top dumps and the community section come * from `community-service.ts`, which the `/community` page reads too. * * @module */ import { db, transaction } from "../db/connection.ts"; import { FRONTEND_URL, PUBLIC_URL } from "../config.ts"; import { DEFAULT_SETTINGS, DIGEST_CADENCES, DIGEST_SETTING, type DigestCadence, PERIOD_MS, } from "../../shared/settings.ts"; import type { CommunityStats } from "../../shared/community.ts"; import type { Dump } from "../../shared/dump.ts"; import { DUMP_SELECT_COLUMNS_ALIASED, dumpRowToApi, isDumpRow, } from "../model/dump.ts"; import { parseRows } from "../model/rows.ts"; import type { Job } from "../lib/scheduler.ts"; import { SETTINGS_COLUMNS } from "../model/settings.ts"; import type { CommentLikedData, DumpCommentedData, DumpRef, NotificationData, NotificationType, PlaylistFollowedData, PlaylistRef, UserFollowedData, UserMentionedData, } from "../../shared/notification.ts"; import { communityPath, dumpPath, feedPath, playlistPath, } from "../../shared/paths.ts"; import { type ActivityKind, type DigestContent, type DigestDump, type DigestHighlight, renderDigestEmail, } from "../mail/digest.ts"; import { lookOf } from "../mail/layout.ts"; import { createUnsubscribeToken } from "../lib/jwt.ts"; import { isEmailEnabled, sendEmail } from "./email-service.ts"; import { getSettings } from "./settings-service.ts"; import { dumpFileVersion } from "./dump-service.ts"; import { withCurrentRefs } from "./notification-service.ts"; import { communityStatsOf } from "./community-service.ts"; const TICK_MS = 15 * 60_000; // What one digest shows at most, as the drafts settled it. const HIGHLIGHTS_MAX = 5; const NETWORK_MAX = 6; const TOP_MAX: Record = { weekly: 5, monthly: 10 }; const MEMBERS_MAX = 10; const PLAYLISTS_MAX = 5; const QUOTE_MAX = 140; const COUNTED: Partial> = { dump_upvoted: "upvotes", user_followed: "followers", dump_commented: "comments", user_mentioned: "mentions", comment_liked: "likes", playlist_followed: "playlistFollows", }; /** A period as the ISO strings dates are stored in, for comparing as text. */ interface Window { from: string; to: string; } /** Flattens Markdown into a short line of text for a quote. */ function quoteOf(markdown: string | undefined | null): string | undefined { if (!markdown) return undefined; const flat = markdown .replace(/!\[[^\]]*\]\([^)]*\)/g, "") .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") .replace(/[`*_>#~]/g, "") .replace(/\s+/g, " ") .trim(); if (!flat) return undefined; return flat.length > QUOTE_MAX ? `${flat.slice(0, QUOTE_MAX - 1).trimEnd()}…` : flat; } /** * Returns the image a mail client can load for `dump`, which must be public: * the thumbnails route for a custom thumbnail or an image or video upload, * else the link preview's image, through the image proxy when it is plain * http, which a mail client may refuse. */ function thumbnailOf(dump: Dump): string | null { const isMedia = dump.kind === "file" && /^(image|video)\//.test(dump.fileMime ?? ""); if (dump.thumbnailMime || isMedia) { return `${PUBLIC_URL}/api/dumps/${dump.id}/thumbnail?v=${ dumpFileVersion(dump) }`; } const thumbnailUrl = dump.richContent?.thumbnailUrl; if (typeof thumbnailUrl !== "string") return null; if (thumbnailUrl.startsWith("https://")) return thumbnailUrl; if (thumbnailUrl.startsWith("http://")) { return `${PUBLIC_URL}/api/remote/image?url=${ encodeURIComponent(thumbnailUrl) }`; } return null; } function toDigestDump(dump: Dump, readerId: string): DigestDump { let domain: string | null = null; if (dump.kind === "url" && dump.url) { try { domain = new URL(dump.url).hostname.replace(/^www\./, ""); } catch { domain = null; } } return { title: dump.title, url: `${FRONTEND_URL}${dumpPath(dump)}`, poster: dump.ownerUsername, domain, thumbnail: thumbnailOf(dump), votes: dump.voteCount, comments: dump.commentCount, mine: dump.userId === readerId, }; } /** * Returns what happened to the reader: their notifications of the period, * counted by kind, and the few worth reading, mentions and comments first. */ function activityOf( readerId: string, w: Window, ): DigestContent["activity"] { const byType = db.prepare( `SELECT type, COUNT(*) AS n FROM notifications WHERE user_id = ? AND created_at >= ? AND created_at < ? GROUP BY type;`, ).all(readerId, w.from, w.to) as { type: NotificationType; n: number }[]; const counts: Partial> = {}; let total = 0; for (const { type, n } of byType) { total += n; const kind = COUNTED[type]; if (kind) counts[kind] = n; } const rows = db.prepare( `SELECT type, data, created_at FROM notifications WHERE user_id = ? AND created_at >= ? AND created_at < ? AND type IN ('user_mentioned', 'dump_commented', 'playlist_followed', 'user_followed', 'comment_liked') ORDER BY CASE type WHEN 'user_mentioned' THEN 0 WHEN 'dump_commented' THEN 0 WHEN 'comment_liked' THEN 2 ELSE 1 END, created_at DESC LIMIT ?;`, ).all(readerId, w.from, w.to, HIGHLIGHTS_MAX) as { type: NotificationType; data: string; created_at: string; }[]; const highlights = rows.flatMap((row) => { try { const h = highlightOf(row.type, withCurrentRefs(JSON.parse(row.data))); return h ? [{ ...h, at: new Date(row.created_at) }] : []; } catch { return []; } }); if (Object.keys(counts).length === 0 && highlights.length === 0) return null; return { counts, highlights, total }; } const commentBody = (id: string) => (db.prepare(`SELECT body FROM comments WHERE id = ? AND deleted = 0;`) .get(id) as { body: string } | undefined)?.body; /** * Returns a highlight's dump, titled `title`, linked where the notification's * joined ref says it is now, or unlinked once it is deleted. */ const dumpTarget = (ref: DumpRef, title: string) => ({ title, url: ref.dumpSlug && ref.dumpOwnerUsername ? `${FRONTEND_URL}${ dumpPath({ ownerUsername: ref.dumpOwnerUsername, slug: ref.dumpSlug }) }` : null, }); /** Returns a highlight's playlist as {@link dumpTarget} does its dump. */ const playlistTarget = (ref: PlaylistRef, title: string) => ({ title, url: ref.playlistSlug && ref.playlistOwnerUsername ? `${FRONTEND_URL}${ playlistPath({ ownerUsername: ref.playlistOwnerUsername, slug: ref.playlistSlug, }) }` : null, }); /** * Returns the highlight a notification makes, from its data with the refs * joined, or null for another kind. */ function highlightOf( type: NotificationType, data: NotificationData, ): Omit | null { // `type` says which member `data` is, which TypeScript can't follow from a // separate argument, so each case names it. switch (type) { case "user_mentioned": { const d = data as UserMentionedData; switch (d.contextType) { case "comment": return { kind: "mentionComment", who: d.mentionerUsername, target: dumpTarget(d, d.contextTitle), quote: quoteOf(commentBody(d.contextId)), }; case "dump": return { kind: "mentionDump", who: d.mentionerUsername, target: dumpTarget(d, d.contextTitle), }; case "playlist": return { kind: "mentionPlaylist", who: d.mentionerUsername, target: playlistTarget(d, d.contextTitle), }; case "chat": return { kind: d.everyone ? "mentionEveryone" : "mentionChat", who: d.mentionerUsername, quote: quoteOf( (db.prepare(`SELECT body FROM chat_messages WHERE id = ?;`) .get(d.contextId) as { body: string } | undefined)?.body, ), }; } return null; } case "dump_commented": { const d = data as DumpCommentedData; return { kind: "comment", who: d.commenterUsername, target: dumpTarget(d, d.dumpTitle), quote: quoteOf(commentBody(d.commentId)), }; } case "playlist_followed": { const d = data as PlaylistFollowedData; return { kind: "playlistFollow", who: d.followerUsername, target: playlistTarget(d, d.playlistTitle), }; } case "user_followed": return { kind: "userFollow", who: (data as UserFollowedData).followerUsername, }; case "comment_liked": { const d = data as CommentLikedData; return { kind: "commentLike", who: d.likerUsername, target: dumpTarget(d, d.dumpTitle), quote: quoteOf(commentBody(d.commentId)), }; } } return null; } /** * Returns the public dumps of the period from the users and playlists the * reader follows, the most upvoted first, without their own. */ function networkOf(readerId: string, w: Window): DigestContent["network"] { const rows = db.prepare( `SELECT ${DUMP_SELECT_COLUMNS_ALIASED}, NULL AS playlist_title, d.created_at AS at FROM dumps d WHERE d.is_private = 0 AND d.user_id != :reader AND d.created_at >= :from AND d.created_at < :to AND d.user_id IN (SELECT followed_user_id FROM follows WHERE follower_id = :reader AND followed_user_id IS NOT NULL) UNION ALL SELECT ${DUMP_SELECT_COLUMNS_ALIASED}, p.title AS playlist_title, pd.added_at AS at FROM playlist_dumps pd JOIN playlists p ON p.id = pd.playlist_id JOIN dumps d ON d.id = pd.dump_id WHERE p.is_public = 1 AND d.is_private = 0 AND d.user_id != :reader AND pd.added_at >= :from AND pd.added_at < :to AND pd.playlist_id IN (SELECT followed_playlist_id FROM follows WHERE follower_id = :reader AND followed_playlist_id IS NOT NULL);`, ).all({ reader: readerId, from: w.from, to: w.to }); const found = parseRows(rows, isDumpRow, (row) => ({ dump: dumpRowToApi(row), playlistTitle: row.playlist_title as string | null, at: row.at as string, })); // A dump both posted by someone followed and added to a followed playlist // is listed once, as the post. const byId = new Map(); for (const item of found) { const seen = byId.get(item.dump.id); if (!seen || (seen.playlistTitle && !item.playlistTitle)) { byId.set(item.dump.id, item); } } const sorted = [...byId.values()].sort((a, b) => b.dump.voteCount - a.dump.voteCount || b.at.localeCompare(a.at) ); if (sorted.length === 0) return null; return { dumps: sorted.slice(0, NETWORK_MAX).map(({ dump, playlistTitle }) => ({ ...toDigestDump(dump, readerId), ...(playlistTitle ? { addedTo: playlistTitle } : {}), })), more: Math.max(0, sorted.length - NETWORK_MAX), }; } /** * Returns the community section from `stats`, the site's figures for the * period, leaving the reader out of who joined. */ function communityOf( readerId: string, stats: CommunityStats, ): DigestContent["community"] { const members = stats.members.filter((m) => m.id !== readerId); const { playlists, chatMessages, totals } = stats; if ( members.length + playlists.length + chatMessages + totals.dumps + totals.upvotes + totals.comments === 0 ) return null; return { members: members.slice(0, MEMBERS_MAX).map((m) => ({ username: m.username, invitedBy: m.invitedByUsername ?? null, })), membersMore: Math.max(0, members.length - MEMBERS_MAX), playlists: playlists.slice(0, PLAYLISTS_MAX).map((p) => ({ title: p.title, url: `${FRONTEND_URL}${playlistPath(p)}`, owner: p.ownerUsername, dumps: p.dumpCount ?? 0, })), playlistsMore: Math.max(0, playlists.length - PLAYLISTS_MAX), chatMessages, totals, }; } /** * Returns the digest of user `readerId` for the period from `from` to `to`, * or null when it would show nothing. The links are absolute; the * unsubscribe one carries `token`, which turns off `cadence`. */ export function buildDigest( readerId: string, reader: string, cadence: DigestCadence, from: Date, to: Date, token: string, ): DigestContent | null { const w = { from: from.toISOString(), to: to.toISOString() }; const weekly = cadence === "weekly"; const activity = weekly ? activityOf(readerId, w) : null; const network = weekly ? networkOf(readerId, w) : null; const stats = communityStatsOf(from, to); const top = stats.top.slice(0, TOP_MAX[cadence]).map((d) => toDigestDump(d, readerId) ); const community = communityOf(readerId, stats); if (!activity && !network && top.length === 0 && !community) return null; return { cadence, from, to, reader, activity, network, top, community, links: { notifications: `${FRONTEND_URL}/notifications`, followed: `${FRONTEND_URL}${feedPath(undefined, "follow")}`, chat: `${FRONTEND_URL}/?chat=open`, community: `${FRONTEND_URL}${communityPath(cadence)}`, settings: `${FRONTEND_URL}/settings`, unsubscribe: `${FRONTEND_URL}/unsubscribe?token=${ encodeURIComponent(token) }`, }, }; } /** * Brings `digest_schedule` in line with the settings: a user who turned a * cadence off loses its row, and one who receives it without a row, being * new or having turned it back on, starts a period at `now`. */ export function syncDigestSchedules(now: Date): void { transaction(() => { for (const cadence of DIGEST_CADENCES) { const setting = DIGEST_SETTING[cadence]; const column = SETTINGS_COLUMNS[setting]; const byDefault = DEFAULT_SETTINGS[setting] ? 1 : 0; db.prepare( `DELETE FROM digest_schedule WHERE cadence = ? AND user_id IN (SELECT user_id FROM user_settings WHERE ${column} = 0);`, ).run(cadence); db.prepare( `INSERT INTO digest_schedule (user_id, cadence, window_start) SELECT u.id, ?, ? FROM users u LEFT JOIN user_settings s ON s.user_id = u.id WHERE COALESCE(s.${column}, ${byDefault}) = 1 AND NOT EXISTS (SELECT 1 FROM digest_schedule ds WHERE ds.user_id = u.id AND ds.cadence = ?);`, ).run(cadence, now.toISOString(), cadence); } }); } /** * Sends every digest due at `now`, one after the other, and returns how many * went out. A digest that fails to send is logged and not retried: its * period is over either way, and the next one starts. */ export async function sendDueDigests(now: Date = new Date()): Promise { if (!isEmailEnabled()) return 0; syncDigestSchedules(now); let sent = 0; for (const cadence of DIGEST_CADENCES) { const period = PERIOD_MS[cadence]; const due = db.prepare( `SELECT ds.user_id, ds.window_start, u.username, u.email FROM digest_schedule ds JOIN users u ON u.id = ds.user_id WHERE ds.cadence = ? AND ds.window_start <= ?;`, ).all(cadence, new Date(now.getTime() - period).toISOString()) as { user_id: string; window_start: string; username: string; email: string; }[]; for (const row of due) { let from = new Date(row.window_start); let to = new Date(from.getTime() + period); // More than a period late (the server was down): cover the last full // period rather than one long gone. if (to.getTime() + period <= now.getTime()) { from = new Date(now.getTime() - period); to = now; } db.prepare( `UPDATE digest_schedule SET window_start = ? WHERE user_id = ? AND cadence = ?;`, ).run(to.toISOString(), row.user_id, cadence); // An address with no dot in its domain, such as the first admin's // admin@localhost, can't receive mail. if (!/@[^@]+\.[^@]+$/.test(row.email)) continue; const token = await createUnsubscribeToken(row.user_id, cadence); const content = buildDigest( row.user_id, row.username, cadence, from, to, token, ); if (!content) continue; const email = renderDigestEmail( content, lookOf(getSettings(row.user_id)), ); try { await sendEmail({ to: row.email, ...email, headers: { "List-Unsubscribe": `<${PUBLIC_URL}/api/unsubscribe?token=${ encodeURIComponent(token) }>`, "List-Unsubscribe-Post": "List-Unsubscribe=One-Click", }, }); sent++; } catch (err) { console.error(`[digest] ${cadence} to ${row.username} failed:`, err); } } } return sent; } /** * Sends the due digests a minute after the server starts, then every quarter * of an hour. `main.ts` starts it. */ export const digestJob: Job = { name: "digest", everyMs: TICK_MS, firstAfterMs: 60_000, async run() { const n = await sendDueDigests(); if (n > 0) console.log(`[digest] sent ${n}`); }, };