gerbeur/api/test/rich-content.test.ts

// Link previews: the HTML extractors, the providers, and fetchWithTimeout's
// guard on every hop. All network access is stubbed.
import {
  createUser,
  get,
  html,
  insertDump,
  PNG,
  stubNetwork,
  test,
} from "./helpers.ts";
import {
  assert,
  assertEquals,
  assertFalse,
  assertRejects,
  assertStringIncludes,
} from "@std/assert";
import {
  extractBestIcon,
  extractFirstContentImage,
  extractJsonLd,
  extractLargeImage,
  extractLinkHref,
  extractMaskIconColor,
  extractMetaName,
  extractOgTag,
  extractPageTitle,
  extractThemeColor,
  fetchRichContent,
  fetchWithTimeout,
  isValidHttpUrl,
  normalizeCssColor,
  readTextCapped,
  tryFetchRichContent,
} from "../services/rich-content-service.ts";

const BASE = "https://example.com/articles/1";

Deno.test("extractOgTag / extractMetaName: either attribute order, entities decoded", () => {
  const page = `
    <meta property="og:title" content="Tom &amp; Jerry &#8212; &#x2764;">
    <meta content='Desc' property='og:description'>
    <meta name="twitter:title" content="TW">
    <meta name="og:title" content="wrong attribute">`;
  assertEquals(extractOgTag(page, "title"), "Tom & Jerry — ❤");
  assertEquals(extractOgTag(page, "description"), "Desc");
  assertEquals(extractOgTag(page, "image"), undefined);
  assertEquals(extractMetaName(page, "twitter:title"), "TW");
});

Deno.test("extractOgTag: content never spans into the next tag", () => {
  const page = `<meta property="og:title"><meta name="x" content="leak">`;
  assertEquals(extractOgTag(page, "title"), undefined);
});

Deno.test("extractPageTitle", () => {
  assertEquals(
    extractPageTitle("<title lang=en>  A &lt;b&gt;  </title>"),
    "A <b>",
  );
  assertEquals(extractPageTitle("<p>none</p>"), undefined);
});

Deno.test("extractThemeColor: unscoped, then light, then any", () => {
  assertEquals(
    extractThemeColor(`
      <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
      <meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`),
    "#fff",
  );
  assertEquals(
    extractThemeColor(`
      <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
      <meta name="theme-color" content="#123">`),
    "#123",
  );
  assertEquals(
    extractThemeColor(
      `<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">`,
    ),
    "#000",
  );
  assertEquals(
    extractMaskIconColor(
      `<link rel="mask-icon" href="/m.svg" color="#5bbad5">`,
    ),
    "#5bbad5",
  );
});

Deno.test("normalizeCssColor", () => {
  assertEquals(normalizeCssColor("#ABC"), "#aabbcc");
  assertEquals(normalizeCssColor("#abcd"), "#aabbcc");
  assertEquals(normalizeCssColor("#112233"), "#112233");
  assertEquals(normalizeCssColor("#11223380"), "#112233");
  assertEquals(normalizeCssColor(" Navy "), "#000080");
  assertEquals(normalizeCssColor("rgb(255, 0, 128)"), "#ff0080");
  assertEquals(normalizeCssColor("rgba(300 -5 50% / .5)"), "#ff0080");
  assertEquals(normalizeCssColor("#12345"), undefined);
  assertEquals(normalizeCssColor("hsl(0 100% 50%)"), undefined);
  assertEquals(normalizeCssColor("rgb(1,2)"), undefined);
  assertEquals(normalizeCssColor(undefined), undefined);
});

Deno.test("extractJsonLd: @graph, arrays and image shapes", () => {
  const ld = (data: unknown) =>
    `<script type="application/ld+json">${JSON.stringify(data)}</script>`;
  assertEquals(
    extractJsonLd(
      ld({
        "@graph": [{ "@type": "WebSite" }, {
          headline: "Head",
          image: { url: "https://i/1.jpg" },
        }],
      }),
    ),
    { title: "Head", description: undefined, thumbnailUrl: "https://i/1.jpg" },
  );
  assertEquals(
    extractJsonLd(ld({ name: ["N"], image: ["https://i/2.jpg"] })).thumbnailUrl,
    "https://i/2.jpg",
  );
  assertEquals(
    extractJsonLd(ld({ name: "N", image: "/relative.jpg" })).thumbnailUrl,
    undefined,
  );
  // Invalid JSON is skipped for the next block.
  assertEquals(
    extractJsonLd(
      `<script type="application/ld+json">{oops</script>${
        ld({ name: "Second" })
      }`,
    ).title,
    "Second",
  );
  assertEquals(extractJsonLd("<p></p>"), {});
});

Deno.test("image extraction: large images, content images, relative URLs", () => {
  const page = `
    <img src="data:image/png;base64,xx" width="500" height="500">
    <img src="/small.jpg" width="50" height="50">
    <img src="/img/site-logo.png">
    <img src="/icons/sprite.svg">
    <img src="/photos/cat.jpg" width="640" height="480">`;
  assertEquals(
    extractLargeImage(page, BASE),
    "https://example.com/photos/cat.jpg",
  );
  assertEquals(
    extractFirstContentImage(page, BASE),
    "https://example.com/small.jpg",
  );
  assertEquals(
    extractFirstContentImage(`<img src="//cdn.test/a.jpg">`, BASE),
    "https://cdn.test/a.jpg",
  );
});

Deno.test("extractBestIcon: SVG beats everything, else the largest", () => {
  assertEquals(
    extractBestIcon(
      `<link rel="icon" href="/16.png" sizes="16x16">
       <link rel="apple-touch-icon" href="/180.png" sizes="180x180">
       <link rel="stylesheet" href="/x.css">`,
      BASE,
    ),
    "https://example.com/180.png",
  );
  assertEquals(
    extractBestIcon(
      `<link rel="icon" href="/180.png" sizes="180x180">
       <link rel="icon" type="image/svg+xml" href="/icon">`,
      BASE,
    ),
    "https://example.com/icon",
  );
  assertEquals(extractBestIcon("<p></p>", BASE), undefined);
  assertEquals(
    extractLinkHref(`<link href="/feed" rel="alternate">`, "alternate", BASE),
    "https://example.com/feed",
  );
});

Deno.test("isValidHttpUrl: http(s) to public hosts, or our own pages", () => {
  assert(isValidHttpUrl("https://example.com/x"));
  assert(isValidHttpUrl("http://gerbeur.test/dumps/abc"));
  assertFalse(isValidHttpUrl("http://localhost:8000/"));
  assertFalse(isValidHttpUrl("http://10.0.0.1/"));
  assertFalse(isValidHttpUrl("file:///etc/passwd"));
  assertFalse(isValidHttpUrl("nope"));
});

Deno.test("fetchWithTimeout: the deadline covers a body that never ends", async () => {
  // Headers at once, then a trickle that never finishes: the read must fail
  // at the deadline rather than hold the request open.
  using _net = stubNetwork((_url, init) =>
    new Response(
      new ReadableStream({
        start(controller) {
          controller.enqueue(new TextEncoder().encode("<html>"));
          init?.signal?.addEventListener(
            "abort",
            () => controller.error(new DOMException("aborted", "AbortError")),
          );
        },
      }),
      { headers: { "content-type": "text/html" } },
    )
  );
  const res = await fetchWithTimeout("https://example.com/slow", 50);
  await assertRejects(() => res.text());
});

Deno.test("tryFetchRichContent: a preview and the post that follows share one fetch", async () => {
  using net = stubNetwork(() => html(`<title>Once</title>`));
  const first = await tryFetchRichContent("https://example.com/once");
  const second = await tryFetchRichContent("https://example.com/once");
  assertEquals(second, first);
  assertEquals(net.calls.length, 1);
  // A refresh always goes to the page.
  await tryFetchRichContent("https://example.com/once", { fresh: true });
  assertEquals(net.calls.length, 2);
});

Deno.test("tryFetchRichContent: failures aren't kept", async () => {
  let up = false;
  using net = stubNetwork(() => {
    if (!up) throw new TypeError("connection refused");
    return html(`<title>Up</title>`);
  });
  const url = "https://example.com/flaky";
  assertFalse((await tryFetchRichContent(url)).ok);
  up = true;
  const { ok, content } = await tryFetchRichContent(url);
  assert(ok);
  assertEquals(content?.title, "Up");
  assertEquals(net.calls.length, 2);
});

Deno.test("readTextCapped: stops reading at the cap", async () => {
  let pulled = 0;
  const body = new ReadableStream<Uint8Array>({
    pull(controller) {
      pulled++;
      controller.enqueue(new TextEncoder().encode("abcdefghij"));
    },
  });
  const text = await readTextCapped(new Response(body), 25);
  assertEquals(text, "abcdefghijabcdefghijabcde");
  assert(pulled < 10);
  assertEquals(await readTextCapped(new Response("short")), "short");
});

Deno.test("fetchWithTimeout: follows redirects, checking every hop", async () => {
  using net = stubNetwork((url) => {
    if (url.pathname === "/start") {
      return new Response(null, { status: 302, headers: { location: "/end" } });
    }
    if (url.pathname === "/to-private") {
      return new Response(null, {
        status: 301,
        headers: { location: "http://169.254.169.254/latest/meta-data" },
      });
    }
    if (url.pathname === "/loop") {
      return new Response(null, {
        status: 302,
        headers: { location: "/loop" },
      });
    }
    return new Response("done");
  });
  const res = await fetchWithTimeout("https://example.com/start");
  assertEquals(await res.text(), "done");
  assertEquals(net.calls.map((u) => u.pathname), ["/start", "/end"]);

  await assertRejects(
    () => fetchWithTimeout("https://example.com/to-private"),
    Error,
    "private",
  );
  await assertRejects(
    () => fetchWithTimeout("https://example.com/loop"),
    TypeError,
    "Too many redirects",
  );
  await assertRejects(
    () => fetchWithTimeout("ftp://example.com/"),
    TypeError,
    "Unsupported protocol",
  );
});

Deno.test("generic provider: the full fallback chain", async () => {
  using _ = stubNetwork(() =>
    html(`<html><head>
      <title>Page title</title>
      <meta property="og:site_name" content="Example">
      <meta property="og:title" content="OG title">
      <meta property="og:image" content="https://example.com/og.jpg">
      <meta name="description" content="Plain description">
      <meta name="theme-color" content="rgb(0, 128, 255)">
      <link rel="icon" href="/favicon-32.png" sizes="32x32">
    </head></html>`)
  );
  const content = await fetchRichContent(BASE);
  assertEquals(content, {
    type: "generic",
    url: BASE,
    title: "OG title",
    description: "Plain description",
    thumbnailUrl: "https://example.com/og.jpg",
    faviconUrl: "https://example.com/favicon-32.png",
    accentColor: "#0080ff",
    siteName: "Example",
  });
});

Deno.test("generic provider: a site-wide og:url means og:title/image are ignored", async () => {
  using _ = stubNetwork(() =>
    html(`
      <meta property="og:url" content="https://example.com/">
      <meta property="og:title" content="Site name">
      <meta property="og:image" content="https://example.com/site.jpg">
      <meta name="twitter:title" content="Article title">
      <img src="/photo.jpg">`)
  );
  const content = await fetchRichContent("https://www.example.com/articles/1");
  assertEquals(content?.title, "Article title");
  assertEquals(content?.thumbnailUrl, "https://www.example.com/photo.jpg");
  assertEquals(content?.siteName, "example.com");
  assertEquals(content?.faviconUrl, "https://www.example.com/favicon.ico");
});

Deno.test("generic provider: non-HTML responses carry just the URL", async () => {
  using _ = stubNetwork(() =>
    new Response("{}", { headers: { "content-type": "application/json" } })
  );
  assertEquals(await fetchRichContent(BASE), { type: "generic", url: BASE });
});

Deno.test("tryFetchRichContent: an unreachable page is ok:false with a stub", async () => {
  using _ = stubNetwork(() => {
    throw new TypeError("connection refused");
  });
  assertEquals(await tryFetchRichContent("https://www.example.com/x"), {
    ok: false,
    content: {
      type: "generic",
      url: "https://www.example.com/x",
      siteName: "example.com",
    },
  });
});

Deno.test("youtube provider: video with start time and playlist", async () => {
  using net = stubNetwork((url) =>
    url.pathname === "/oembed"
      ? Response.json({ title: "Never Gonna", thumbnail_url: "ignored" })
      : new Response(null, { status: 404 })
  );
  const content = await fetchRichContent(
    "https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PL123&t=1m30s",
  );
  assertEquals(content?.type, "youtube");
  assertEquals(content?.videoId, "dQw4w9WgXcQ");
  assertEquals(content?.title, "Never Gonna");
  assertEquals(
    content?.thumbnailUrl,
    "https://img.youtube.com/vi/dQw4w9WgXcQ/hqdefault.jpg",
  );
  assertEquals(
    content?.embedUrl,
    "https://www.youtube.com/embed/dQw4w9WgXcQ?rel=0&list=PL123&start=90",
  );
  assertEquals(net.calls[0].hostname, "www.youtube.com");
});

Deno.test("youtube provider: youtu.be, shorts, playlists and channels", async () => {
  using _ = stubNetwork((url) =>
    url.pathname === "/oembed"
      ? Response.json({
        title: "T",
        thumbnail_url: "https://i.ytimg.com/x.jpg",
      })
      : html(`<meta property="og:title" content="Channel">
              <meta property="og:image" content="https://yt3/ch.jpg">`)
  );
  assertEquals(
    (await fetchRichContent("https://youtu.be/abc123?t=42"))?.embedUrl,
    "https://www.youtube.com/embed/abc123?rel=0&start=42",
  );
  assertEquals(
    (await fetchRichContent("https://youtube.com/shorts/xyz"))?.videoId,
    "xyz",
  );
  const list = await fetchRichContent(
    "https://www.youtube.com/playlist?list=PLabc",
  );
  assertEquals(
    list?.embedUrl,
    "https://www.youtube.com/embed/videoseries?list=PLabc&rel=0",
  );
  assertEquals(list?.thumbnailUrl, "https://i.ytimg.com/x.jpg");
  const channel = await fetchRichContent("https://www.youtube.com/@someone");
  assertEquals(channel?.title, "Channel");
  assertEquals(channel?.embedUrl, undefined);
});

Deno.test("bandcamp and soundcloud providers", async () => {
  using _ = stubNetwork(() =>
    html(`<meta property="og:title" content="Album">
          <meta property="og:video" content="https://bandcamp.com/EmbeddedPlayer/album=1">`)
  );
  const bc = await fetchRichContent("https://artist.bandcamp.com/album/a");
  assertEquals(bc?.type, "bandcamp");
  assertEquals(bc?.title, "Album");
  assertEquals(bc?.embedUrl, "https://bandcamp.com/EmbeddedPlayer/album=1");
  const sc = await fetchRichContent("https://soundcloud.com/artist/track");
  assertEquals(sc?.type, "soundcloud");
  assertStringIncludes(
    sc?.embedUrl ?? "",
    encodeURIComponent("https://soundcloud.com/artist/track"),
  );
});

test("self provider: our own dump, user and playlist pages come from the DB", async () => {
  const user = await createUser();
  const dump = insertDump({
    userId: user.id,
    title: "Local dump",
    comment: "About it",
    richContent: { type: "generic", url: "x", thumbnailUrl: "https://t/x.jpg" },
  });
  using net = stubNetwork(() => {
    throw new Error("no network for self links");
  });
  const content = await fetchRichContent(
    `http://gerbeur.test/dumps/${dump.slug}`,
  );
  assertEquals(content?.title, "Local dump");
  assertEquals(content?.description, "About it");
  assertEquals(content?.thumbnailUrl, "https://t/x.jpg");
  assertEquals(content?.siteName, "gerbeur");
  const profile = await fetchRichContent(
    `http://gerbeur.test/users/${user.username}`,
  );
  assertEquals(profile?.title, user.username);
  assertEquals(net.calls.length, 0);
});

Deno.test("GET /api/proxy-image: rasters only, fetched once for everyone", async () => {
  using net = stubNetwork((url) =>
    url.pathname.endsWith(".svg")
      ? new Response("<svg/>", { headers: { "content-type": "image/svg+xml" } })
      : new Response(PNG, { headers: { "content-type": "image/png" } })
  );
  const path = `/api/proxy-image?url=${
    encodeURIComponent("https://cdn.example.com/a.png")
  }`;
  const first = await get(path);
  assertEquals(first.status, 200);
  assertEquals(first.headers.get("content-type"), "image/png");
  assertEquals(first.bytes, PNG);
  const second = await get(path);
  assertEquals(second.bytes, PNG);
  assertEquals(net.calls.length, 1);

  const svg = await get(
    `/api/proxy-image?url=${
      encodeURIComponent("https://cdn.example.com/a.svg")
    }`,
  );
  assertEquals(svg.status, 400);
});