import { html, stubNetwork } from "../helpers.ts";
import { assert, assertEquals, assertFalse, assertRejects } from "@std/assert";
import {
fetchPage,
fetchWithTimeout,
isValidHttpUrl,
readTextCapped,
} from "../../remote/fetch.ts";
Deno.test("isValidHttpUrl: http(s) to public hosts, or our own pages", () => {
assert(isValidHttpUrl("https://example.com/x"));
assert(isValidHttpUrl("http://gerbeur.test/dumps/abc"));
assertFalse(isValidHttpUrl("http://localhost:8000/"));
assertFalse(isValidHttpUrl("http://10.0.0.1/"));
assertFalse(isValidHttpUrl("file:///etc/passwd"));
assertFalse(isValidHttpUrl("nope"));
});
Deno.test("fetchWithTimeout: the deadline covers a body that never ends", async () => {
// Headers at once, then a trickle that never finishes: the read must fail
// at the deadline rather than hold the request open.
using _net = stubNetwork((_url, init) =>
new Response(
new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(""));
init?.signal?.addEventListener(
"abort",
() => controller.error(new DOMException("aborted", "AbortError")),
);
},
}),
{ headers: { "content-type": "text/html" } },
)
);
const res = await fetchWithTimeout("https://example.com/slow", 50);
await assertRejects(() => res.text());
});
Deno.test("readTextCapped: stops reading at the cap", async () => {
let pulled = 0;
const body = new ReadableStream({
pull(controller) {
pulled++;
controller.enqueue(new TextEncoder().encode("abcdefghij"));
},
});
const text = await readTextCapped(new Response(body), 25);
assertEquals(text, "abcdefghijabcdefghijabcde");
assert(pulled < 10);
assertEquals(await readTextCapped(new Response("short")), "short");
});
Deno.test("fetchWithTimeout: follows redirects, checking every hop", async () => {
using net = stubNetwork((url) => {
if (url.pathname === "/start") {
return new Response(null, { status: 302, headers: { location: "/end" } });
}
if (url.pathname === "/to-private") {
return new Response(null, {
status: 301,
headers: { location: "http://169.254.169.254/latest/meta-data" },
});
}
if (url.pathname === "/loop") {
return new Response(null, {
status: 302,
headers: { location: "/loop" },
});
}
return new Response("done");
});
const res = await fetchWithTimeout("https://example.com/start");
assertEquals(await res.text(), "done");
assertEquals(net.calls.map((u) => u.pathname), ["/start", "/end"]);
await assertRejects(
() => fetchWithTimeout("https://example.com/to-private"),
Error,
"private",
);
await assertRejects(
() => fetchWithTimeout("https://example.com/loop"),
TypeError,
"Too many redirects",
);
await assertRejects(
() => fetchWithTimeout("ftp://example.com/"),
TypeError,
"Unsupported protocol",
);
});
Deno.test("readTextCapped: decodes the charset the header or a declares", async () => {
const cafe = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // "café" in Latin-1
const page = (contentType: string, head = "") =>
new Response(
new Uint8Array([...new TextEncoder().encode(head), ...cafe]),
{ headers: { "content-type": contentType } },
);
assertEquals(
await readTextCapped(page("text/html; charset=ISO-8859-1")),
"café",
);
assertEquals(
await readTextCapped(page("text/html", '')),
'café',
);
assertEquals(
await readTextCapped(page("text/html; charset=nonsense")),
"caf�",
);
});
Deno.test("fetchPage: the page and where it was served from", async () => {
using _net = stubNetwork((url) => {
if (url.pathname === "/moved") {
return new Response(null, {
status: 301,
headers: { location: "https://www.example.com/here" },
});
}
if (url.pathname === "/gone") return html("Gone", 410);
if (url.pathname === "/data") {
return new Response("{}", {
headers: { "content-type": "application/json" },
});
}
return html("Here");
});
assertEquals(await fetchPage("https://example.com/moved"), {
url: "https://www.example.com/here",
html: "Here",
});
assertEquals(await fetchPage("https://example.com/data"), {
url: "https://example.com/data",
});
await assertRejects(
() => fetchPage("https://example.com/gone"),
Error,
"410",
);
});