gerbeur/api/test/remote/html.test.ts

import "../setup.ts";
import { assertEquals } from "@std/assert";
import {
  absoluteHttpUrl,
  decodeHtmlEntities,
  normalizeCssColor,
  parsePage,
} from "../../remote/html.ts";

const BASE = "https://example.com/articles/1";

const read = (html: string) => parsePage(html, BASE);

Deno.test("parsePage meta: either attribute, either order, entities decoded", () => {
  const page = read(`
    <meta property="og:title" content="Tom &amp; Jerry &#8212; &#x2764;">
    <meta content='Desc' property='og:description'>
    <meta name="twitter:title" content="TW">
    <meta name="og:image" content="https://example.com/by-name.jpg">
    <meta property="og:title" content="second, ignored">`);
  assertEquals(page.meta("og:title"), "Tom & Jerry — ❤");
  assertEquals(page.meta("og:description"), "Desc");
  assertEquals(page.meta("twitter:title"), "TW");
  // Sites write og: tags under `name` too.
  assertEquals(page.meta("og:image"), "https://example.com/by-name.jpg");
  assertEquals(page.meta("OG:Title"), "Tom & Jerry — ❤");
  assertEquals(page.meta("og:url"), undefined);
});

Deno.test("parsePage meta: attributes read as a browser reads them", () => {
  const page = read(`
    <meta property=og:title content=Unquoted>
    <meta property="og:description" content="a > b, still the tag">
    <meta data-property="og:image" content="https://example.com/no.jpg">
    <meta property="og:site_name"><meta name="x" content="leak">
    <meta property="og:type" content="  ">`);
  assertEquals(page.meta("og:title"), "Unquoted");
  assertEquals(page.meta("og:description"), "a > b, still the tag");
  assertEquals(page.meta("og:image"), undefined);
  assertEquals(page.meta("og:site_name"), undefined);
  // A blank value is no value, so a fallback can take over.
  assertEquals(page.meta("og:type"), undefined);
});

Deno.test("parsePage: tags in comments and scripts aren't read", () => {
  const page = read(`
    <!-- <meta property="og:title" content="commented"> -->
    <script>document.write('<meta property="og:title" content="scripted">')</script>
    <meta property="og:title" content="Real">`);
  assertEquals(page.meta("og:title"), "Real");
  // An unclosed comment runs to the end of the page.
  assertEquals(
    read(`<!-- <meta property="og:title" content="x">`).meta("og:title"),
    undefined,
  );
});

Deno.test("parsePage title", () => {
  assertEquals(read("<title lang=en>  A &lt;b&gt;  </title>").title, "A <b>");
  assertEquals(read("<title>  </title>").title, undefined);
  assertEquals(read("<p>none</p>").title, undefined);
});

Deno.test("decodeHtmlEntities: one pass, the names pages use, nothing thrown", () => {
  assertEquals(
    decodeHtmlEntities("Caf&eacute; &Agrave; l&rsquo;&eacute;t&eacute;"),
    "Café À l’été",
  );
  assertEquals(
    decodeHtmlEntities("&laquo;&nbsp;Oui&hellip;&raquo;"),
    "« Oui…»",
  );
  assertEquals(decodeHtmlEntities("&ccedil;a &AMP; &Ntilde;"), "ça & Ñ");
  // `&amp;quot;` is a literal `&quot;` on the page.
  assertEquals(decodeHtmlEntities("&amp;quot;"), "&quot;");
  // No such code point or name: left as written.
  assertEquals(decodeHtmlEntities("&#99999999;"), "&#99999999;");
  assertEquals(decodeHtmlEntities("&#xD800;"), "&#xD800;");
  assertEquals(decodeHtmlEntities("&bogus;"), "&bogus;");
});

Deno.test("absoluteHttpUrl", () => {
  assertEquals(absoluteHttpUrl("/og.png", BASE), "https://example.com/og.png");
  assertEquals(
    absoluteHttpUrl("//cdn.test/a.jpg", BASE),
    "https://cdn.test/a.jpg",
  );
  assertEquals(
    absoluteHttpUrl(" og.png ", BASE),
    "https://example.com/articles/og.png",
  );
  assertEquals(absoluteHttpUrl("javascript:alert(1)", BASE), undefined);
  assertEquals(absoluteHttpUrl("data:image/png;base64,xx", BASE), undefined);
  assertEquals(absoluteHttpUrl("http://[bad", BASE), undefined);
  assertEquals(absoluteHttpUrl("", BASE), undefined);
});

Deno.test("parsePage themeColor: unscoped, then light, then any", () => {
  assertEquals(
    read(`
      <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
      <meta name="theme-color" media="(prefers-color-scheme: light)" content="#fff">`)
      .themeColor,
    "#fff",
  );
  assertEquals(
    read(`
      <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">
      <meta name="theme-color" content="#123">`).themeColor,
    "#123",
  );
  assertEquals(
    read(
      `<meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000">`,
    ).themeColor,
    "#000",
  );
  assertEquals(
    read(`<link rel="mask-icon" href="/m.svg" color="#5bbad5">`).maskIconColor,
    "#5bbad5",
  );
});

Deno.test("normalizeCssColor", () => {
  assertEquals(normalizeCssColor("#ABC"), "#aabbcc");
  assertEquals(normalizeCssColor("#abcd"), "#aabbcc");
  assertEquals(normalizeCssColor("#112233"), "#112233");
  assertEquals(normalizeCssColor("#11223380"), "#112233");
  assertEquals(normalizeCssColor(" Navy "), "#000080");
  assertEquals(normalizeCssColor("rgb(255, 0, 128)"), "#ff0080");
  assertEquals(normalizeCssColor("rgba(300 -5 50% / .5)"), "#ff0080");
  assertEquals(normalizeCssColor("#12345"), undefined);
  assertEquals(normalizeCssColor("hsl(0 100% 50%)"), undefined);
  assertEquals(normalizeCssColor("rgb(1,2)"), undefined);
  assertEquals(normalizeCssColor("transparent"), undefined);
  assertEquals(normalizeCssColor(undefined), undefined);
});

Deno.test("parsePage jsonLd: @graph, arrays and image shapes", () => {
  const ld = (data: unknown) =>
    `<script type="application/ld+json">${JSON.stringify(data)}</script>`;
  assertEquals(
    read(ld({
      "@graph": [{ "@type": "WebSite" }, {
        headline: "Head",
        image: { url: "https://i/1.jpg" },
      }],
    })).jsonLd,
    { title: "Head", description: undefined, thumbnailUrl: "https://i/1.jpg" },
  );
  assertEquals(
    read(ld({ name: ["N"], image: ["https://i/2.jpg"] })).jsonLd.thumbnailUrl,
    "https://i/2.jpg",
  );
  assertEquals(
    read(ld({ name: "N", image: "/relative.jpg" })).jsonLd.thumbnailUrl,
    "https://example.com/relative.jpg",
  );
  // Invalid JSON is skipped for the next block.
  assertEquals(
    read(
      `<script type="application/ld+json">{oops</script>${
        ld({ name: "Second" })
      }`,
    ).jsonLd.title,
    "Second",
  );
  assertEquals(read("<p></p>").jsonLd, {});
});

Deno.test("parsePage jsonLd: the page's node beats the site's", () => {
  const ld = (data: unknown) =>
    `<script type="application/ld+json">${JSON.stringify(data)}</script>`;
  const page = read(
    ld({ "@type": "Organization", name: "Acme", logo: "https://a/l.png" }) +
      ld({
        "@graph": [
          { "@type": ["WebSite"], name: "Acme site" },
          { "@type": "NewsArticle", headline: "The news" },
        ],
      }),
  );
  assertEquals(page.jsonLd.title, "The news");
  // With nothing else, the site's node still names the page.
  assertEquals(
    read(ld({ "@type": "Organization", name: "Acme" })).jsonLd.title,
    "Acme",
  );
});

Deno.test("parsePage images: large images, content images, relative URLs", () => {
  const page = read(`
    <img src="data:image/png;base64,xx" width="500" height="500">
    <img src="/pixel.gif" width="1" height="1">
    <img src="/small.jpg" width="60" height="60">
    <img src="/img/site-logo.png">
    <img src="/icons/sprite.svg">
    <img src="/photos/cat.jpg" width="640" height="480">`);
  assertEquals(page.largeImage, "https://example.com/photos/cat.jpg");
  assertEquals(page.firstContentImage, "https://example.com/small.jpg");
  assertEquals(
    read(`<img src="//cdn.test/a.jpg">`).firstContentImage,
    "https://cdn.test/a.jpg",
  );
});

Deno.test("parsePage images: lazy images by their data-src", () => {
  const page = read(`
    <img data-src="/wrong.jpg" src="/img/lazy-placeholder.gif">
    <img src="data:image/gif;base64,R0lGOD" data-src="/real.jpg" width="800" height="600">`);
  assertEquals(page.largeImage, "https://example.com/real.jpg");
  assertEquals(page.firstContentImage, "https://example.com/real.jpg");
});

Deno.test("parsePage bestIcon: SVG beats everything, else the largest", () => {
  assertEquals(
    read(`<link rel="icon" href="/16.png" sizes="16x16">
       <link rel="apple-touch-icon" href="/180.png" sizes="180x180">
       <link rel="stylesheet" href="/x.css">`).bestIcon,
    "https://example.com/180.png",
  );
  assertEquals(
    read(`<link rel="icon" href="/180.png" sizes="180x180">
       <link rel="icon" type="image/svg+xml" href="/icon">`).bestIcon,
    "https://example.com/icon",
  );
  assertEquals(
    read(`<link rel="shortcut icon" href="/f.ico" sizes="16x16 48x48">
       <link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
    "https://example.com/f.ico",
  );
  assertEquals(read("<p></p>").bestIcon, undefined);
});

Deno.test("parsePage bestIcon: a mask icon is never the icon", () => {
  assertEquals(
    read(`<link rel="mask-icon" href="/pinned.svg" color="#000">
       <link rel="icon" href="/32.png" sizes="32x32">`).bestIcon,
    "https://example.com/32.png",
  );
});