547 lines
18 KiB
TypeScript
547 lines
18 KiB
TypeScript
/**
|
|
* Sends each user their weekly digest and monthly recap, unless they turned
|
|
* it off. `main.ts` starts {@link digestJob}, which every quarter of an hour
|
|
* sends the digests that are due; `digest_schedule` remembers, for
|
|
* each user and cadence, the start of the period their next one covers.
|
|
*
|
|
* A period is 7 days or 30, rolling from when the user's schedule began, so
|
|
* the digests spread over the week instead of leaving together. A digest
|
|
* covers the period just ended, or the last full period when the server was
|
|
* down longer than one; a digest with nothing to show isn't sent, and either
|
|
* way the next period begins. The top dumps and the community section come
|
|
* from `community-service.ts`, which the `/community` page reads too.
|
|
*
|
|
* @module
|
|
*/
|
|
|
|
import { db, transaction } from "../db/connection.ts";
|
|
import { FRONTEND_URL, PUBLIC_URL } from "../config.ts";
|
|
import {
|
|
DEFAULT_SETTINGS,
|
|
DIGEST_CADENCES,
|
|
DIGEST_SETTING,
|
|
type DigestCadence,
|
|
PERIOD_MS,
|
|
} from "../../shared/settings.ts";
|
|
import type { CommunityStats } from "../../shared/community.ts";
|
|
import type { Dump } from "../../shared/dump.ts";
|
|
import {
|
|
DUMP_SELECT_COLUMNS_ALIASED,
|
|
dumpRowToApi,
|
|
isDumpRow,
|
|
} from "../model/dump.ts";
|
|
import { parseRows } from "../model/rows.ts";
|
|
import type { Job } from "../lib/scheduler.ts";
|
|
import { SETTINGS_COLUMNS } from "../model/settings.ts";
|
|
import type {
|
|
CommentLikedData,
|
|
DumpCommentedData,
|
|
DumpRef,
|
|
NotificationData,
|
|
NotificationType,
|
|
PlaylistFollowedData,
|
|
PlaylistRef,
|
|
UserFollowedData,
|
|
UserMentionedData,
|
|
} from "../../shared/notification.ts";
|
|
import {
|
|
communityPath,
|
|
dumpPath,
|
|
feedPath,
|
|
playlistPath,
|
|
} from "../../shared/paths.ts";
|
|
import {
|
|
type ActivityKind,
|
|
type DigestContent,
|
|
type DigestDump,
|
|
type DigestHighlight,
|
|
renderDigestEmail,
|
|
} from "../mail/digest.ts";
|
|
import { lookOf } from "../mail/layout.ts";
|
|
import { createUnsubscribeToken } from "../lib/jwt.ts";
|
|
import { isEmailEnabled, sendEmail } from "./email-service.ts";
|
|
import { getSettings } from "./settings-service.ts";
|
|
import { dumpFileVersion } from "./dump-service.ts";
|
|
import { withCurrentRefs } from "./notification-service.ts";
|
|
import { communityStatsOf } from "./community-service.ts";
|
|
|
|
const TICK_MS = 15 * 60_000;
|
|
|
|
// What one digest shows at most, as the drafts settled it.
|
|
const HIGHLIGHTS_MAX = 5;
|
|
const NETWORK_MAX = 6;
|
|
const TOP_MAX: Record<DigestCadence, number> = { weekly: 5, monthly: 10 };
|
|
const MEMBERS_MAX = 10;
|
|
const PLAYLISTS_MAX = 5;
|
|
const QUOTE_MAX = 140;
|
|
|
|
const COUNTED: Partial<Record<NotificationType, ActivityKind>> = {
|
|
dump_upvoted: "upvotes",
|
|
user_followed: "followers",
|
|
dump_commented: "comments",
|
|
user_mentioned: "mentions",
|
|
comment_liked: "likes",
|
|
playlist_followed: "playlistFollows",
|
|
};
|
|
|
|
/** A period as the ISO strings dates are stored in, for comparing as text. */
|
|
interface Window {
|
|
from: string;
|
|
to: string;
|
|
}
|
|
|
|
/** Flattens Markdown into a short line of text for a quote. */
|
|
function quoteOf(markdown: string | undefined | null): string | undefined {
|
|
if (!markdown) return undefined;
|
|
const flat = markdown
|
|
.replace(/!\[[^\]]*\]\([^)]*\)/g, "")
|
|
.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
|
|
.replace(/[`*_>#~]/g, "")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
if (!flat) return undefined;
|
|
return flat.length > QUOTE_MAX
|
|
? `${flat.slice(0, QUOTE_MAX - 1).trimEnd()}…`
|
|
: flat;
|
|
}
|
|
|
|
/**
|
|
* Returns the image a mail client can load for `dump`, which must be public:
|
|
* the thumbnails route for a custom thumbnail or an image or video upload,
|
|
* else the link preview's image, through the image proxy when it is plain
|
|
* http, which a mail client may refuse.
|
|
*/
|
|
function thumbnailOf(dump: Dump): string | null {
|
|
const isMedia = dump.kind === "file" &&
|
|
/^(image|video)\//.test(dump.fileMime ?? "");
|
|
if (dump.thumbnailMime || isMedia) {
|
|
return `${PUBLIC_URL}/api/dumps/${dump.id}/thumbnail?v=${
|
|
dumpFileVersion(dump)
|
|
}`;
|
|
}
|
|
const thumbnailUrl = dump.richContent?.thumbnailUrl;
|
|
if (typeof thumbnailUrl !== "string") return null;
|
|
if (thumbnailUrl.startsWith("https://")) return thumbnailUrl;
|
|
if (thumbnailUrl.startsWith("http://")) {
|
|
return `${PUBLIC_URL}/api/remote/image?url=${
|
|
encodeURIComponent(thumbnailUrl)
|
|
}`;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function toDigestDump(dump: Dump, readerId: string): DigestDump {
|
|
let domain: string | null = null;
|
|
if (dump.kind === "url" && dump.url) {
|
|
try {
|
|
domain = new URL(dump.url).hostname.replace(/^www\./, "");
|
|
} catch {
|
|
domain = null;
|
|
}
|
|
}
|
|
return {
|
|
title: dump.title,
|
|
url: `${FRONTEND_URL}${dumpPath(dump)}`,
|
|
poster: dump.ownerUsername,
|
|
domain,
|
|
thumbnail: thumbnailOf(dump),
|
|
votes: dump.voteCount,
|
|
comments: dump.commentCount,
|
|
mine: dump.userId === readerId,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Returns what happened to the reader: their notifications of the period,
|
|
* counted by kind, and the few worth reading, mentions and comments first.
|
|
*/
|
|
function activityOf(
|
|
readerId: string,
|
|
w: Window,
|
|
): DigestContent["activity"] {
|
|
const byType = db.prepare(
|
|
`SELECT type, COUNT(*) AS n FROM notifications
|
|
WHERE user_id = ? AND created_at >= ? AND created_at < ?
|
|
GROUP BY type;`,
|
|
).all(readerId, w.from, w.to) as { type: NotificationType; n: number }[];
|
|
const counts: Partial<Record<ActivityKind, number>> = {};
|
|
let total = 0;
|
|
for (const { type, n } of byType) {
|
|
total += n;
|
|
const kind = COUNTED[type];
|
|
if (kind) counts[kind] = n;
|
|
}
|
|
|
|
const rows = db.prepare(
|
|
`SELECT type, data, created_at FROM notifications
|
|
WHERE user_id = ? AND created_at >= ? AND created_at < ?
|
|
AND type IN ('user_mentioned', 'dump_commented', 'playlist_followed',
|
|
'user_followed', 'comment_liked')
|
|
ORDER BY CASE type
|
|
WHEN 'user_mentioned' THEN 0 WHEN 'dump_commented' THEN 0
|
|
WHEN 'comment_liked' THEN 2 ELSE 1 END,
|
|
created_at DESC
|
|
LIMIT ?;`,
|
|
).all(readerId, w.from, w.to, HIGHLIGHTS_MAX) as {
|
|
type: NotificationType;
|
|
data: string;
|
|
created_at: string;
|
|
}[];
|
|
const highlights = rows.flatMap((row) => {
|
|
try {
|
|
const h = highlightOf(row.type, withCurrentRefs(JSON.parse(row.data)));
|
|
return h ? [{ ...h, at: new Date(row.created_at) }] : [];
|
|
} catch {
|
|
return [];
|
|
}
|
|
});
|
|
|
|
if (Object.keys(counts).length === 0 && highlights.length === 0) return null;
|
|
return { counts, highlights, total };
|
|
}
|
|
|
|
const commentBody = (id: string) =>
|
|
(db.prepare(`SELECT body FROM comments WHERE id = ? AND deleted = 0;`)
|
|
.get(id) as { body: string } | undefined)?.body;
|
|
/**
|
|
* Returns a highlight's dump, titled `title`, linked where the notification's
|
|
* joined ref says it is now, or unlinked once it is deleted.
|
|
*/
|
|
const dumpTarget = (ref: DumpRef, title: string) => ({
|
|
title,
|
|
url: ref.dumpSlug && ref.dumpOwnerUsername
|
|
? `${FRONTEND_URL}${
|
|
dumpPath({ ownerUsername: ref.dumpOwnerUsername, slug: ref.dumpSlug })
|
|
}`
|
|
: null,
|
|
});
|
|
/** Returns a highlight's playlist as {@link dumpTarget} does its dump. */
|
|
const playlistTarget = (ref: PlaylistRef, title: string) => ({
|
|
title,
|
|
url: ref.playlistSlug && ref.playlistOwnerUsername
|
|
? `${FRONTEND_URL}${
|
|
playlistPath({
|
|
ownerUsername: ref.playlistOwnerUsername,
|
|
slug: ref.playlistSlug,
|
|
})
|
|
}`
|
|
: null,
|
|
});
|
|
|
|
/**
|
|
* Returns the highlight a notification makes, from its data with the refs
|
|
* joined, or null for another kind.
|
|
*/
|
|
function highlightOf(
|
|
type: NotificationType,
|
|
data: NotificationData,
|
|
): Omit<DigestHighlight, "at"> | null {
|
|
// `type` says which member `data` is, which TypeScript can't follow from a
|
|
// separate argument, so each case names it.
|
|
switch (type) {
|
|
case "user_mentioned": {
|
|
const d = data as UserMentionedData;
|
|
switch (d.contextType) {
|
|
case "comment":
|
|
return {
|
|
kind: "mentionComment",
|
|
who: d.mentionerUsername,
|
|
target: dumpTarget(d, d.contextTitle),
|
|
quote: quoteOf(commentBody(d.contextId)),
|
|
};
|
|
case "dump":
|
|
return {
|
|
kind: "mentionDump",
|
|
who: d.mentionerUsername,
|
|
target: dumpTarget(d, d.contextTitle),
|
|
};
|
|
case "playlist":
|
|
return {
|
|
kind: "mentionPlaylist",
|
|
who: d.mentionerUsername,
|
|
target: playlistTarget(d, d.contextTitle),
|
|
};
|
|
case "chat":
|
|
return {
|
|
kind: "mentionChat",
|
|
who: d.mentionerUsername,
|
|
quote: quoteOf(
|
|
(db.prepare(`SELECT body FROM chat_messages WHERE id = ?;`)
|
|
.get(d.contextId) as { body: string } | undefined)?.body,
|
|
),
|
|
};
|
|
}
|
|
return null;
|
|
}
|
|
case "dump_commented": {
|
|
const d = data as DumpCommentedData;
|
|
return {
|
|
kind: "comment",
|
|
who: d.commenterUsername,
|
|
target: dumpTarget(d, d.dumpTitle),
|
|
quote: quoteOf(commentBody(d.commentId)),
|
|
};
|
|
}
|
|
case "playlist_followed": {
|
|
const d = data as PlaylistFollowedData;
|
|
return {
|
|
kind: "playlistFollow",
|
|
who: d.followerUsername,
|
|
target: playlistTarget(d, d.playlistTitle),
|
|
};
|
|
}
|
|
case "user_followed":
|
|
return {
|
|
kind: "userFollow",
|
|
who: (data as UserFollowedData).followerUsername,
|
|
};
|
|
case "comment_liked": {
|
|
const d = data as CommentLikedData;
|
|
return {
|
|
kind: "commentLike",
|
|
who: d.likerUsername,
|
|
target: dumpTarget(d, d.dumpTitle),
|
|
quote: quoteOf(commentBody(d.commentId)),
|
|
};
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Returns the public dumps of the period from the users and playlists the
|
|
* reader follows, the most upvoted first, without their own.
|
|
*/
|
|
function networkOf(readerId: string, w: Window): DigestContent["network"] {
|
|
const rows = db.prepare(
|
|
`SELECT ${DUMP_SELECT_COLUMNS_ALIASED}, NULL AS playlist_title, d.created_at AS at
|
|
FROM dumps d
|
|
WHERE d.is_private = 0 AND d.user_id != :reader
|
|
AND d.created_at >= :from AND d.created_at < :to
|
|
AND d.user_id IN (SELECT followed_user_id FROM follows
|
|
WHERE follower_id = :reader AND followed_user_id IS NOT NULL)
|
|
UNION ALL
|
|
SELECT ${DUMP_SELECT_COLUMNS_ALIASED}, p.title AS playlist_title, pd.added_at AS at
|
|
FROM playlist_dumps pd
|
|
JOIN playlists p ON p.id = pd.playlist_id
|
|
JOIN dumps d ON d.id = pd.dump_id
|
|
WHERE p.is_public = 1 AND d.is_private = 0 AND d.user_id != :reader
|
|
AND pd.added_at >= :from AND pd.added_at < :to
|
|
AND pd.playlist_id IN (SELECT followed_playlist_id FROM follows
|
|
WHERE follower_id = :reader AND followed_playlist_id IS NOT NULL);`,
|
|
).all({ reader: readerId, from: w.from, to: w.to });
|
|
const found = parseRows(rows, isDumpRow, (row) => ({
|
|
dump: dumpRowToApi(row),
|
|
playlistTitle: row.playlist_title as string | null,
|
|
at: row.at as string,
|
|
}));
|
|
// A dump both posted by someone followed and added to a followed playlist
|
|
// is listed once, as the post.
|
|
const byId = new Map<string, (typeof found)[number]>();
|
|
for (const item of found) {
|
|
const seen = byId.get(item.dump.id);
|
|
if (!seen || (seen.playlistTitle && !item.playlistTitle)) {
|
|
byId.set(item.dump.id, item);
|
|
}
|
|
}
|
|
const sorted = [...byId.values()].sort((a, b) =>
|
|
b.dump.voteCount - a.dump.voteCount || b.at.localeCompare(a.at)
|
|
);
|
|
if (sorted.length === 0) return null;
|
|
return {
|
|
dumps: sorted.slice(0, NETWORK_MAX).map(({ dump, playlistTitle }) => ({
|
|
...toDigestDump(dump, readerId),
|
|
...(playlistTitle ? { addedTo: playlistTitle } : {}),
|
|
})),
|
|
more: Math.max(0, sorted.length - NETWORK_MAX),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Returns the community section from `stats`, the site's figures for the
|
|
* period, leaving the reader out of who joined.
|
|
*/
|
|
function communityOf(
|
|
readerId: string,
|
|
stats: CommunityStats,
|
|
): DigestContent["community"] {
|
|
const members = stats.members.filter((m) => m.id !== readerId);
|
|
const { playlists, chatMessages, totals } = stats;
|
|
if (
|
|
members.length + playlists.length + chatMessages + totals.dumps +
|
|
totals.upvotes + totals.comments === 0
|
|
) return null;
|
|
return {
|
|
members: members.slice(0, MEMBERS_MAX).map((m) => ({
|
|
username: m.username,
|
|
invitedBy: m.invitedByUsername ?? null,
|
|
})),
|
|
membersMore: Math.max(0, members.length - MEMBERS_MAX),
|
|
playlists: playlists.slice(0, PLAYLISTS_MAX).map((p) => ({
|
|
title: p.title,
|
|
url: `${FRONTEND_URL}${playlistPath(p)}`,
|
|
owner: p.ownerUsername,
|
|
dumps: p.dumpCount ?? 0,
|
|
})),
|
|
playlistsMore: Math.max(0, playlists.length - PLAYLISTS_MAX),
|
|
chatMessages,
|
|
totals,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Returns the digest of user `readerId` for the period from `from` to `to`,
|
|
* or null when it would show nothing. The links are absolute; the
|
|
* unsubscribe one carries `token`, which turns off `cadence`.
|
|
*/
|
|
export function buildDigest(
|
|
readerId: string,
|
|
reader: string,
|
|
cadence: DigestCadence,
|
|
from: Date,
|
|
to: Date,
|
|
token: string,
|
|
): DigestContent | null {
|
|
const w = { from: from.toISOString(), to: to.toISOString() };
|
|
const weekly = cadence === "weekly";
|
|
const activity = weekly ? activityOf(readerId, w) : null;
|
|
const network = weekly ? networkOf(readerId, w) : null;
|
|
const stats = communityStatsOf(from, to);
|
|
const top = stats.top.slice(0, TOP_MAX[cadence]).map((d) =>
|
|
toDigestDump(d, readerId)
|
|
);
|
|
const community = communityOf(readerId, stats);
|
|
if (!activity && !network && top.length === 0 && !community) return null;
|
|
return {
|
|
cadence,
|
|
from,
|
|
to,
|
|
reader,
|
|
activity,
|
|
network,
|
|
top,
|
|
community,
|
|
links: {
|
|
notifications: `${FRONTEND_URL}/notifications`,
|
|
followed: `${FRONTEND_URL}${feedPath(undefined, "follow")}`,
|
|
chat: `${FRONTEND_URL}/?chat=open`,
|
|
community: `${FRONTEND_URL}${communityPath(cadence)}`,
|
|
settings: `${FRONTEND_URL}/settings`,
|
|
unsubscribe: `${FRONTEND_URL}/unsubscribe?token=${
|
|
encodeURIComponent(token)
|
|
}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Brings `digest_schedule` in line with the settings: a user who turned a
|
|
* cadence off loses its row, and one who receives it without a row, being
|
|
* new or having turned it back on, starts a period at `now`.
|
|
*/
|
|
export function syncDigestSchedules(now: Date): void {
|
|
transaction(() => {
|
|
for (const cadence of DIGEST_CADENCES) {
|
|
const setting = DIGEST_SETTING[cadence];
|
|
const column = SETTINGS_COLUMNS[setting];
|
|
const byDefault = DEFAULT_SETTINGS[setting] ? 1 : 0;
|
|
db.prepare(
|
|
`DELETE FROM digest_schedule WHERE cadence = ? AND user_id IN
|
|
(SELECT user_id FROM user_settings WHERE ${column} = 0);`,
|
|
).run(cadence);
|
|
db.prepare(
|
|
`INSERT INTO digest_schedule (user_id, cadence, window_start)
|
|
SELECT u.id, ?, ? FROM users u
|
|
LEFT JOIN user_settings s ON s.user_id = u.id
|
|
WHERE COALESCE(s.${column}, ${byDefault}) = 1
|
|
AND NOT EXISTS (SELECT 1 FROM digest_schedule ds
|
|
WHERE ds.user_id = u.id AND ds.cadence = ?);`,
|
|
).run(cadence, now.toISOString(), cadence);
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Sends every digest due at `now`, one after the other, and returns how many
|
|
* went out. A digest that fails to send is logged and not retried: its
|
|
* period is over either way, and the next one starts.
|
|
*/
|
|
export async function sendDueDigests(now: Date = new Date()): Promise<number> {
|
|
if (!isEmailEnabled()) return 0;
|
|
syncDigestSchedules(now);
|
|
let sent = 0;
|
|
for (const cadence of DIGEST_CADENCES) {
|
|
const period = PERIOD_MS[cadence];
|
|
const due = db.prepare(
|
|
`SELECT ds.user_id, ds.window_start, u.username, u.email
|
|
FROM digest_schedule ds JOIN users u ON u.id = ds.user_id
|
|
WHERE ds.cadence = ? AND ds.window_start <= ?;`,
|
|
).all(cadence, new Date(now.getTime() - period).toISOString()) as {
|
|
user_id: string;
|
|
window_start: string;
|
|
username: string;
|
|
email: string;
|
|
}[];
|
|
for (const row of due) {
|
|
let from = new Date(row.window_start);
|
|
let to = new Date(from.getTime() + period);
|
|
// More than a period late (the server was down): cover the last full
|
|
// period rather than one long gone.
|
|
if (to.getTime() + period <= now.getTime()) {
|
|
from = new Date(now.getTime() - period);
|
|
to = now;
|
|
}
|
|
db.prepare(
|
|
`UPDATE digest_schedule SET window_start = ? WHERE user_id = ? AND cadence = ?;`,
|
|
).run(to.toISOString(), row.user_id, cadence);
|
|
// An address with no dot in its domain, such as the first admin's
|
|
// admin@localhost, can't receive mail.
|
|
if (!/@[^@]+\.[^@]+$/.test(row.email)) continue;
|
|
const token = await createUnsubscribeToken(row.user_id, cadence);
|
|
const content = buildDigest(
|
|
row.user_id,
|
|
row.username,
|
|
cadence,
|
|
from,
|
|
to,
|
|
token,
|
|
);
|
|
if (!content) continue;
|
|
const email = renderDigestEmail(
|
|
content,
|
|
lookOf(getSettings(row.user_id)),
|
|
);
|
|
try {
|
|
await sendEmail({
|
|
to: row.email,
|
|
...email,
|
|
headers: {
|
|
"List-Unsubscribe": `<${PUBLIC_URL}/api/unsubscribe?token=${
|
|
encodeURIComponent(token)
|
|
}>`,
|
|
"List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
|
|
},
|
|
});
|
|
sent++;
|
|
} catch (err) {
|
|
console.error(`[digest] ${cadence} to ${row.username} failed:`, err);
|
|
}
|
|
}
|
|
}
|
|
return sent;
|
|
}
|
|
|
|
/**
|
|
* Sends the due digests a minute after the server starts, then every quarter
|
|
* of an hour. `main.ts` starts it.
|
|
*/
|
|
export const digestJob: Job = {
|
|
name: "digest",
|
|
everyMs: TICK_MS,
|
|
firstAfterMs: 60_000,
|
|
async run() {
|
|
const n = await sendDueDigests();
|
|
if (n > 0) console.log(`[digest] sent ${n}`);
|
|
},
|
|
};
|