// Checksums of text and files against the ones given, and of this site against its manifest; Node runs all above page()
// DIGITS hex digits by algorithm · STRENGTH weakest to strongest · LENGTHS algorithm by digits · HEX hex of a byte
// LIMIT bytes "compare with" or a list file may hold · MOST files a manifest or a choice may hold · utf8 encoder
// no throw an Error with the properties of its second argument · num en-US number · plural "3 bytes"
// which the algorithm a tag names, or null · NEITHER a refusal · GNU TAGGED checksum line forms · entry one line read
// VARIANTS the slips a text is tried with · pause a turn for the page · unsafe what sitePath refuses
// createHash selfTest from sha.js
export {toHex, digest, algorithmFor, groups, formatBytes, formatSize, displayName, parseDigest, digestText, parseList,
parseExpected, isListFileName, parseSidecar, mergeLists, comparePaths, formatList, matchFiles, compareDigests,
otherAlgorithms, invisibles, textVariants, findVariant, hashSource, sitePath, decodeManifest, parseManifest,
checkEntries, parseColophon, ledgerVerdict, checkSite, sameBytes};
import {createHash, selfTest} from "./sha.js";
const DIGITS = {"SHA-1": 40, "SHA-256": 64, "SHA-384": 96, "SHA-512": 128}, STRENGTH = Object.keys(DIGITS);
const LENGTHS = Object.fromEntries(Object.entries(DIGITS).map(([k, n]) => [n, k]));
const HEX = Array.from({length: 2 ** 8}, (_, i) => i.toString(16).padStart(2, "0"));
const LIMIT = 1e6, MOST = 1e4, utf8 = new TextEncoder();
const no = (s, o) => { throw Object.assign(new Error(s), o) };
const num = n => String(n).replace(/\B(?=(\d{3})+$)/g, ","), plural = (n, s) => `${num(n)} ${n === 1 ? s : s + "s"}`;
function toHex(buffer) {
let s = "";
for (const x of new Uint8Array(buffer)) s += HEX[x];
return s }
async function digest(algorithm, data) {
return toHex(await crypto.subtle.digest(algorithm, typeof data === "string" ? utf8.encode(data) : data)) }
function algorithmFor(hex) {
return LENGTHS[hex.length] || no(hex.length === 32 ? "32 digits is usually MD5, which this page doesn't compute."
: `No algorithm here has ${hex.length}-digit digests.`) }
function which(s) {
const k = s.toUpperCase().replace(/^SHA2?-?(?=\d{3}$)/, "SHA-").replace(/^SHA-?1$/, "SHA-1");
return DIGITS[k] ? k : null }
const groups = hex => hex.match(/.{1,8}/g) ?? [];
const formatBytes = n => plural(n, "byte");
// inv: the unit is picked after rounding: 999,999 bytes is 1.0 MB, not 1000 kB
function formatSize(n) {
if (n < 1e3) return formatBytes(n);
const f = j => { const x = n / 1e3**j; return x < 99.95 ? x.toFixed(1) : String(Math.round(x)) };
let i = Math.min(Math.floor(Math.log10(n) / 3), 4);
if (i < 4 && +f(i) >= 1e3) i++;
return `${f(i)} ${["", "kB", "MB", "GB", "TB"][i]}` }
// Unicode: "invoice\u202Efdp.exe" displays as invoiceexe.pdf
const displayName = name => name.replace(/[\0-\x1f\x7f-\x9f\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]/g, c => `\\u{${c.charCodeAt(0).toString(16).toUpperCase().padStart(4, "0")}}`);
const NEITHER = "That's neither a digest nor a checksum list.";
function parseDigest(text) {
const t = text.trim(), m = /^([a-z0-9-]+):([0-9a-f]+)$/i.exec(t);
if (m) {
const algorithm = which(m[1]), n = m[2].length;
algorithm || no(`That's ${m[1]}, which this page doesn't compute.`);
n !== DIGITS[algorithm] && no(`A ${algorithm} digest has ${DIGITS[algorithm]} digits; this one has ${n}.`);
return {algorithm, hexes: [m[2].toLowerCase()], notation: "oci", named: true} }
const a = t.split(/\s+/);
if (a.every(s => /^[0-9a-f]+$/i.test(s))) {
const s = a.join("").toLowerCase();
return {algorithm: algorithmFor(s), hexes: [s], notation: "hex", named: false} }
a.every(s => /^[a-z]+\d*-/i.test(s)) || no(NEITHER);
// SRI: the strongest algorithm present counts and any of its values may match; options after "?" are printable ASCII
// guard: one optional ?-group covers "?a?b"; a repeated one backtracks over every split of a run of "?"
const r = a.flatMap(x => {
const [, k, v] = /^([a-z0-9]+)-([A-Za-z0-9+/]+={0,2})(?:\?[\x21-\x7e]*)?$/i.exec(x) ?? [];
k || no(NEITHER);
const algorithm = /^sha(256|384|512)$/i.test(k) && which(k);
if (!algorithm) { a.length === 1 && no(`SRI values use sha256, sha384 or sha512, not ${k}.`); return [] }
let s = "";
try { s = atob(v) } catch {}
s.length !== DIGITS[algorithm] / 2 && no(`An SRI ${k} value holds ${DIGITS[algorithm] / 2} bytes; this one holds ${s.length}.`);
return [{algorithm, hex: Array.from(s, c => HEX[c.charCodeAt(0)]).join("")}] });
const algorithm = STRENGTH.findLast(k => r.some(x => x.algorithm === k));
algorithm || no("SRI values use sha256, sha384 or sha512.");
return {algorithm, hexes: r.filter(x => x.algorithm === algorithm).map(x => x.hex), notation: "sri", named: true} }
// guard: written from the parse; text riding along in a link never shows beside a digest that isn't the one in use
function digestText({algorithm, hexes, notation}) {
const k = algorithm.replace("-", "").toLowerCase();
const f = s => btoa(String.fromCharCode(...s.match(/../g).map(x => parseInt(x, 16))));
return notation === "sri" ? hexes.map(s => `${k}-${f(s)}`).join(" ")
: notation === "oci" ? `${k}:${hexes[0]}` : hexes[0] }
const GNU = /^(\\?)([0-9a-f]+) [ *]?(.+)$/is, TAGGED = /^(\\?)([a-z][\w/-]*) ?\((.*)\) ?= ?([0-9a-f]+)$/is;
function entry(s, n) {
const o = {line: n};
let m = GNU.exec(s), q, k, hex, p, algorithm;
if (m) [, q, hex, p] = m;
else if ((m = TAGGED.exec(s))) [, q, k, p, hex] = m;
else no(`Line ${n} isn't a checksum line.`, o);
hex = hex.toLowerCase();
if (k) {
algorithm = which(k);
algorithm || no(`Line ${n} uses ${k}, which this page doesn't compute.`, o);
hex.length !== DIGITS[algorithm] && no(`Line ${n} has a ${hex.length}-digit digest; ${algorithm} has ${DIGITS[algorithm]}.`, o) }
else try { algorithm = algorithmFor(hex) } catch (w) { no(`Line ${n}: ${w.message}`, o) }
// GNU coreutils: a name holding a backslash or a line break is escaped, and its line starts with a backslash
const path = q ? p.replace(/\\(.?)/gs, (_, c) => ({"\\": "\\", n: "\n", r: "\r"})[c] ?? no(`Line ${n} isn't a checksum line.`, o)) : p;
return {algorithm, hex, path, line: n, named: !!k} }
function parseList(text) {
const a = text.replace(/^\uFEFF/, "").split("\n").map(s => s.replace(/\r$/, "")), j = a.findIndex(s => s.trim());
let i = 0, n = a.length, signed = false;
// OpenPGP: armor headers end at the first blank line; signed means a signature follows, which nothing here checks
if (a[j]?.trim() === "-----BEGIN PGP SIGNED MESSAGE-----") {
i = j + 1;
while (i < n && a[i] !== "") i++;
n = a.indexOf("-----BEGIN PGP SIGNATURE-----", i);
n < 0 && no("The list has a PGP header but no signature.", {line: j + 1});
signed = true }
const entries = [], seen = new Map();
for (; i < n; i++) {
const s = signed && a[i].startsWith("- ") ? a[i].slice(2) : a[i];
if (!s.trim() || /^\s*#/.test(s)) continue;
const r = entry(s, i + 1), k = `${r.algorithm}\0${r.path}`, y = seen.get(k);
if (!y) { seen.set(k, r); entries.push(r) }
else y.hex !== r.hex && no(`Lines ${y.line} and ${r.line} give ${displayName(r.path)} two different digests.`, {line: r.line}) }
entries.length || no("The list names no files.");
return {entries, signed} }
function parseExpected(text) {
// UTF-8: a UTF-16 unit takes 1 to 3 bytes
(text.length > LIMIT || (text.length > LIMIT / 3 && utf8.encode(text).length > LIMIT)) && no("That's longer than any checksum list this page reads (1 MB).");
const t = text.replace(/^\uFEFF/, "").trim();
if (!t) return {kind: "none"};
if (!/[\r\n]/.test(t)) {
const a = t.split(/\s+/), q = a.every(s => /^[0-9a-f]+$/i.test(s)) && !(a.length > 1 && LENGTHS[a[0].length]);
if (a.length === 1 || q || a.every(s => /^[a-z]+\d*-\S+$/i.test(s))) return {kind: "digest", ...parseDigest(t)} }
const o = parseList(text), [r] = o.entries;
// GNU coreutils, OpenSSL: sha256sum and openssl dgst name standard input - or stdin, and one such line is a digest
return o.entries.length === 1 && (r.path === "-" || r.path === "stdin") && !o.signed
? {kind: "digest", algorithm: r.algorithm, hexes: [r.hex], notation: "hex", named: r.named} : {kind: "list", ...o} }
const isListFileName = name => /^(?:sha(?:1|256|384|512)sums?(?:\.txt)?|shasums(?:1|256|384|512)?(?:\.txt)?|checksums?(?:\.txt)?|manifest\.txt|.+-checksum|.+\.sha(?:1|256|384|512)(?:sum)?)(?:\.asc)?$/i.test(name);
function parseSidecar(fileName, text) {
const p = /^(.+)\.sha(?:1|256|384|512)(?:sum)?$/i.exec(fileName)?.[1], s = text.replace(/^\uFEFF/, "").trim();
return p && /^[0-9a-f]+$/i.test(s) ? [{algorithm: algorithmFor(s), hex: s.toLowerCase(), path: p, line: 1, named: false}]
: parseList(text).entries }
// guard: list files chosen together are one list, and a path two of them give two digests refuses with both named
function mergeLists(lists) {
const entries = [], seen = new Map();
for (const x of lists) for (const r of x.entries) {
const k = `${r.algorithm}\0${r.path}`, y = seen.get(k);
if (!y) { seen.set(k, {r, name: x.name}); entries.push(r) }
else y.r.hex !== r.hex && no(`${y.name} and ${x.name} give ${displayName(r.path)} two different digests.`) }
return {entries, signed: lists.some(x => x.signed)} }
// JS: sort() compares UTF-16 units; build.py and LC_ALL=C sort compare code points
function comparePaths(a, b) {
const x = [...a], y = [...b];
for (let i = 0; i < x.length && i < y.length; i++) if (x[i] !== y[i]) return x[i].codePointAt(0) - y[i].codePointAt(0);
return x.length - y.length }
function formatList(items) {
return [...items].sort((x, y) => comparePaths(x.path, y.path)).map(({hex, path}) => /[\\\n\r]/.test(path)
? `\\${hex} ${path.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/\r/g, "\\r")}\n` : `${hex} ${path}\n`).join("") }
function matchFiles(entries, files) {
const rest = new Map();
for (const r of entries) {
const y = rest.get(r.path);
if (!y || STRENGTH.indexOf(r.algorithm) > STRENGTH.indexOf(y.algorithm)) rest.set(r.path, r) }
const bare = p => p.replace(/^(\.\/)+/, ""), nfc = p => bare(p).normalize("NFC");
const rel = x => x.folder && x.path.startsWith(x.folder + "/") ? x.path.slice(x.folder.length + 1) : x.path;
const base = p => p.slice(p.lastIndexOf("/") + 1).normalize("NFC");
const pairs = [], ambiguous = [], left = new Set(files), dup = new Set();
for (const [how, f, g] of [
["exact", r => r.path, x => [x.path]],
["dot-slash", r => bare(r.path), x => [bare(x.path)]],
["folder", r => bare(r.path), x => [bare(rel(x))]],
["nfc", r => nfc(r.path), x => [nfc(x.path), nfc(rel(x))]],
["basename", r => base(r.path), x => [base(x.path)]]]) {
const by = new Map(), own = new Map();
for (const r of rest.values()) { const k = f(r); by.set(k, by.has(k) ? null : r) }
for (const x of left) {
const r = g(x).map(k => by.get(k)).find(Boolean);
if (r) own.set(r, [...(own.get(r) ?? []), x]) }
for (const [r, a] of own) {
rest.delete(r.path);
for (const x of a) left.delete(x);
if (a.length === 1) pairs.push({entry: r, file: a[0], how});
else { ambiguous.push({entry: r, files: a}); for (const x of a) dup.add(x) } } }
return {pairs, missing: [...rest.values()], unlisted: [...left].filter(x => !dup.has(x)), ambiguous} }
function compareDigests(expectedHex, actualHex) {
const x = expectedHex.toLowerCase(), y = actualHex.toLowerCase(), positions = [];
if (x.length !== y.length) return {verdict: "differs", differing: Math.max(x.length, y.length), positions};
for (let i = 0; i < x.length; i++) if (x[i] !== y[i]) positions.push(i);
const n = positions.length;
return {verdict: n === 0 ? "match" : n <= 4 ? "typo" : "differs", differing: n, positions} }
const otherAlgorithms = length => ({64: ["SHA3-256", "BLAKE3"], 96: ["SHA3-384"], 128: ["SHA3-512", "BLAKE2b"]})[length] ?? [];
function invisibles(text) {
let breaks = 0;
while (text[text.length-1-breaks] === "\n") breaks++;
const f = re => text.match(re)?.length ?? 0, bom = text.startsWith("\uFEFF");
const nbsp = f(/[\u00a0\u2007\u202f]/g);
// Unicode: a Default_Ignorable_Code_Point draws as nothing; the direction marks among them have a count of their own
const zero = f(/(?![\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069])\p{Default_Ignorable_Code_Point}/gu) - (bom ? 1 : 0);
const marks = f(/[\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]/g);
const lone = f(/[\ud800-\udbff](?![\udc00-\udfff])|(? s + "\n"],
["the line break at the end removed", s => s.replace(/\n$/, "")],
["Windows line breaks (CRLF)", s => s.replace(/\n/g, "\r\n")],
["Windows line breaks and one at the end", s => s.replace(/\n?$/, "\n").replace(/\n/g, "\r\n")],
["its accents composed (Unicode NFC)", s => s.normalize("NFC")],
["its accents decomposed (Unicode NFD)", s => s.normalize("NFD")],
["the spaces and line breaks at either end trimmed", s => s.trim()],
["a byte order mark at the start", s => "\uFEFF" + s]];
// inv: variants are told apart by their UTF-8, where a lone surrogate is U+FFFD
function textVariants(text) {
if (text.length > 2 ** 20) return [];
const back = new TextDecoder("utf-8", {ignoreBOM: true}), seen = new Set([back.decode(utf8.encode(text))]);
return VARIANTS.flatMap(([label, f]) => {
const bytes = utf8.encode(f(text)), k = back.decode(bytes);
if (seen.has(k)) return [];
seen.add(k);
return [{label, bytes}] }) }
async function findVariant(text, {algorithm, hexes}) {
for (const {label, bytes} of textVariants(text)) if (hexes.includes(await digest(algorithm, bytes))) return {label};
return null }
const pause = () => globalThis.scheduler?.yield?.() ?? new Promise(f => setTimeout(f, 0));
async function hashSource(source, algorithms, {chunk = 2 ** 20, signal, onProgress} = {}) {
const a = algorithms.map(createHash), {size} = source;
const abort = () => new DOMException("Hashing was stopped.", "AbortError");
if (signal?.aborted) throw abort();
let done = 0, t = Date.now(), next = size ? source.read(0, Math.min(chunk, size)) : null;
while (done < size) {
const b = await next;
if (signal?.aborted) throw abort();
b.length !== Math.min(chunk, size - done) && no("The file changed while it was read.");
done += b.length;
next = done < size ? source.read(done, Math.min(done + chunk, size)) : null;
for (const x of a) x.update(b);
onProgress?.(done, size);
if (Date.now() - t >= 4) { await pause(); t = Date.now() } }
if (!size) onProgress?.(0, 0);
return Object.fromEntries(algorithms.map((k, i) => [k, toHex(a[i].digest())])) }
const unsafe = p => !p || /[\0-\x1f\x7f-\x9f]/.test(p) || p.split("/").some(s => s === "" || s === "." || s === "..");
function sitePath(path) {
unsafe(path) && no(`${displayName(path)} isn't a safe path.`);
return "/" + path.split("/").map(encodeURIComponent).join("/") }
function decodeManifest(bytes) {
try { return new TextDecoder("utf-8", {fatal: true}).decode(bytes) } catch { no("manifest.txt isn't valid UTF-8.") } }
// guard: an unreadable line refuses the list; skipping it would pass nonsense as zero files, all matching
function parseManifest(text) {
const o = [], seen = new Map(), a = text.split("\n");
for (let i = 0; i < a.length; i++) {
const n = i + 1, at = {line: n};
if (a[i].trim() === "") continue;
const [, digest, path] = /^([0-9a-f]+) [ *](.+)$/s.exec(a[i]) ?? [];
digest || no(`Line ${n} isn't a checksum line.`, at);
try { algorithmFor(digest) } catch (w) { no(`Line ${n}: ${w.message}`, at) }
unsafe(path) && no(`Line ${n} names an unsafe path.`, at);
seen.has(path) && no(`Lines ${seen.get(path)} and ${n} both name ${displayName(path)}.`, at);
seen.set(path, n);
o.push({digest, path, line: n}) > MOST && no(`It lists more than ${num(MOST)} files.`, at) }
o.length || no("The manifest lists no files.");
return o }
async function checkEntries(entries, load, {concurrency = 6, onProgress} = {}) {
const o = [];
let next = 0, done = 0;
await Promise.all(Array.from({length: Math.min(concurrency, entries.length)}, async () => {
while (next < entries.length) {
const i = next++, r = entries[i];
try {
const b = await load(r.path), actual = await digest(algorithmFor(r.digest), b), ok = actual === r.digest;
o[i] = {...r, actual, size: b.byteLength, ok, status: ok ? "ok" : "changed"} }
catch (w) {
o[i] = {...r, actual: null, ok: false, status: w?.status === 404 ? "missing" : "unreadable", error: String(w)} }
onProgress?.(++done, entries.length) } }));
return o }
function parseColophon(html) {
const m = /