// Checksums of text and files against the ones given, and of this site against its manifest; Node runs all above page() // DIGITS hex digits by algorithm · STRENGTH weakest to strongest · LENGTHS algorithm by digits · HEX hex of a byte // LIMIT bytes "compare with" or a list file may hold · MOST files a manifest or a choice may hold · utf8 encoder // no throw an Error with the properties of its second argument · num en-US number · plural "3 bytes" // which the algorithm a tag names, or null · NEITHER a refusal · GNU TAGGED checksum line forms · entry one line read // VARIANTS the slips a text is tried with · pause a turn for the page · unsafe what sitePath refuses // createHash selfTest from sha.js export {toHex, digest, algorithmFor, groups, formatBytes, formatSize, displayName, parseDigest, digestText, parseList, parseExpected, isListFileName, parseSidecar, mergeLists, comparePaths, formatList, matchFiles, compareDigests, otherAlgorithms, invisibles, textVariants, findVariant, hashSource, sitePath, decodeManifest, parseManifest, checkEntries, parseColophon, ledgerVerdict, checkSite, sameBytes}; import {createHash, selfTest} from "./sha.js"; const DIGITS = {"SHA-1": 40, "SHA-256": 64, "SHA-384": 96, "SHA-512": 128}, STRENGTH = Object.keys(DIGITS); const LENGTHS = Object.fromEntries(Object.entries(DIGITS).map(([k, n]) => [n, k])); const HEX = Array.from({length: 2 ** 8}, (_, i) => i.toString(16).padStart(2, "0")); const LIMIT = 1e6, MOST = 1e4, utf8 = new TextEncoder(); const no = (s, o) => { throw Object.assign(new Error(s), o) }; const num = n => String(n).replace(/\B(?=(\d{3})+$)/g, ","), plural = (n, s) => `${num(n)} ${n === 1 ? s : s + "s"}`; function toHex(buffer) { let s = ""; for (const x of new Uint8Array(buffer)) s += HEX[x]; return s } async function digest(algorithm, data) { return toHex(await crypto.subtle.digest(algorithm, typeof data === "string" ? utf8.encode(data) : data)) } function algorithmFor(hex) { return LENGTHS[hex.length] || no(hex.length === 32 ? "32 digits is usually MD5, which this page doesn't compute." : `No algorithm here has ${hex.length}-digit digests.`) } function which(s) { const k = s.toUpperCase().replace(/^SHA2?-?(?=\d{3}$)/, "SHA-").replace(/^SHA-?1$/, "SHA-1"); return DIGITS[k] ? k : null } const groups = hex => hex.match(/.{1,8}/g) ?? []; const formatBytes = n => plural(n, "byte"); // inv: the unit is picked after rounding: 999,999 bytes is 1.0 MB, not 1000 kB function formatSize(n) { if (n < 1e3) return formatBytes(n); const f = j => { const x = n / 1e3**j; return x < 99.95 ? x.toFixed(1) : String(Math.round(x)) }; let i = Math.min(Math.floor(Math.log10(n) / 3), 4); if (i < 4 && +f(i) >= 1e3) i++; return `${f(i)} ${["", "kB", "MB", "GB", "TB"][i]}` } // Unicode: "invoice\u202Efdp.exe" displays as invoiceexe.pdf const displayName = name => name.replace(/[\0-\x1f\x7f-\x9f\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]/g, c => `\\u{${c.charCodeAt(0).toString(16).toUpperCase().padStart(4, "0")}}`); const NEITHER = "That's neither a digest nor a checksum list."; function parseDigest(text) { const t = text.trim(), m = /^([a-z0-9-]+):([0-9a-f]+)$/i.exec(t); if (m) { const algorithm = which(m[1]), n = m[2].length; algorithm || no(`That's ${m[1]}, which this page doesn't compute.`); n !== DIGITS[algorithm] && no(`A ${algorithm} digest has ${DIGITS[algorithm]} digits; this one has ${n}.`); return {algorithm, hexes: [m[2].toLowerCase()], notation: "oci", named: true} } const a = t.split(/\s+/); if (a.every(s => /^[0-9a-f]+$/i.test(s))) { const s = a.join("").toLowerCase(); return {algorithm: algorithmFor(s), hexes: [s], notation: "hex", named: false} } a.every(s => /^[a-z]+\d*-/i.test(s)) || no(NEITHER); // SRI: the strongest algorithm present counts and any of its values may match; options after "?" are printable ASCII // guard: one optional ?-group covers "?a?b"; a repeated one backtracks over every split of a run of "?" const r = a.flatMap(x => { const [, k, v] = /^([a-z0-9]+)-([A-Za-z0-9+/]+={0,2})(?:\?[\x21-\x7e]*)?$/i.exec(x) ?? []; k || no(NEITHER); const algorithm = /^sha(256|384|512)$/i.test(k) && which(k); if (!algorithm) { a.length === 1 && no(`SRI values use sha256, sha384 or sha512, not ${k}.`); return [] } let s = ""; try { s = atob(v) } catch {} s.length !== DIGITS[algorithm] / 2 && no(`An SRI ${k} value holds ${DIGITS[algorithm] / 2} bytes; this one holds ${s.length}.`); return [{algorithm, hex: Array.from(s, c => HEX[c.charCodeAt(0)]).join("")}] }); const algorithm = STRENGTH.findLast(k => r.some(x => x.algorithm === k)); algorithm || no("SRI values use sha256, sha384 or sha512."); return {algorithm, hexes: r.filter(x => x.algorithm === algorithm).map(x => x.hex), notation: "sri", named: true} } // guard: written from the parse; text riding along in a link never shows beside a digest that isn't the one in use function digestText({algorithm, hexes, notation}) { const k = algorithm.replace("-", "").toLowerCase(); const f = s => btoa(String.fromCharCode(...s.match(/../g).map(x => parseInt(x, 16)))); return notation === "sri" ? hexes.map(s => `${k}-${f(s)}`).join(" ") : notation === "oci" ? `${k}:${hexes[0]}` : hexes[0] } const GNU = /^(\\?)([0-9a-f]+) [ *]?(.+)$/is, TAGGED = /^(\\?)([a-z][\w/-]*) ?\((.*)\) ?= ?([0-9a-f]+)$/is; function entry(s, n) { const o = {line: n}; let m = GNU.exec(s), q, k, hex, p, algorithm; if (m) [, q, hex, p] = m; else if ((m = TAGGED.exec(s))) [, q, k, p, hex] = m; else no(`Line ${n} isn't a checksum line.`, o); hex = hex.toLowerCase(); if (k) { algorithm = which(k); algorithm || no(`Line ${n} uses ${k}, which this page doesn't compute.`, o); hex.length !== DIGITS[algorithm] && no(`Line ${n} has a ${hex.length}-digit digest; ${algorithm} has ${DIGITS[algorithm]}.`, o) } else try { algorithm = algorithmFor(hex) } catch (w) { no(`Line ${n}: ${w.message}`, o) } // GNU coreutils: a name holding a backslash or a line break is escaped, and its line starts with a backslash const path = q ? p.replace(/\\(.?)/gs, (_, c) => ({"\\": "\\", n: "\n", r: "\r"})[c] ?? no(`Line ${n} isn't a checksum line.`, o)) : p; return {algorithm, hex, path, line: n, named: !!k} } function parseList(text) { const a = text.replace(/^\uFEFF/, "").split("\n").map(s => s.replace(/\r$/, "")), j = a.findIndex(s => s.trim()); let i = 0, n = a.length, signed = false; // OpenPGP: armor headers end at the first blank line; signed means a signature follows, which nothing here checks if (a[j]?.trim() === "-----BEGIN PGP SIGNED MESSAGE-----") { i = j + 1; while (i < n && a[i] !== "") i++; n = a.indexOf("-----BEGIN PGP SIGNATURE-----", i); n < 0 && no("The list has a PGP header but no signature.", {line: j + 1}); signed = true } const entries = [], seen = new Map(); for (; i < n; i++) { const s = signed && a[i].startsWith("- ") ? a[i].slice(2) : a[i]; if (!s.trim() || /^\s*#/.test(s)) continue; const r = entry(s, i + 1), k = `${r.algorithm}\0${r.path}`, y = seen.get(k); if (!y) { seen.set(k, r); entries.push(r) } else y.hex !== r.hex && no(`Lines ${y.line} and ${r.line} give ${displayName(r.path)} two different digests.`, {line: r.line}) } entries.length || no("The list names no files."); return {entries, signed} } function parseExpected(text) { // UTF-8: a UTF-16 unit takes 1 to 3 bytes (text.length > LIMIT || (text.length > LIMIT / 3 && utf8.encode(text).length > LIMIT)) && no("That's longer than any checksum list this page reads (1 MB)."); const t = text.replace(/^\uFEFF/, "").trim(); if (!t) return {kind: "none"}; if (!/[\r\n]/.test(t)) { const a = t.split(/\s+/), q = a.every(s => /^[0-9a-f]+$/i.test(s)) && !(a.length > 1 && LENGTHS[a[0].length]); if (a.length === 1 || q || a.every(s => /^[a-z]+\d*-\S+$/i.test(s))) return {kind: "digest", ...parseDigest(t)} } const o = parseList(text), [r] = o.entries; // GNU coreutils, OpenSSL: sha256sum and openssl dgst name standard input - or stdin, and one such line is a digest return o.entries.length === 1 && (r.path === "-" || r.path === "stdin") && !o.signed ? {kind: "digest", algorithm: r.algorithm, hexes: [r.hex], notation: "hex", named: r.named} : {kind: "list", ...o} } const isListFileName = name => /^(?:sha(?:1|256|384|512)sums?(?:\.txt)?|shasums(?:1|256|384|512)?(?:\.txt)?|checksums?(?:\.txt)?|manifest\.txt|.+-checksum|.+\.sha(?:1|256|384|512)(?:sum)?)(?:\.asc)?$/i.test(name); function parseSidecar(fileName, text) { const p = /^(.+)\.sha(?:1|256|384|512)(?:sum)?$/i.exec(fileName)?.[1], s = text.replace(/^\uFEFF/, "").trim(); return p && /^[0-9a-f]+$/i.test(s) ? [{algorithm: algorithmFor(s), hex: s.toLowerCase(), path: p, line: 1, named: false}] : parseList(text).entries } // guard: list files chosen together are one list, and a path two of them give two digests refuses with both named function mergeLists(lists) { const entries = [], seen = new Map(); for (const x of lists) for (const r of x.entries) { const k = `${r.algorithm}\0${r.path}`, y = seen.get(k); if (!y) { seen.set(k, {r, name: x.name}); entries.push(r) } else y.r.hex !== r.hex && no(`${y.name} and ${x.name} give ${displayName(r.path)} two different digests.`) } return {entries, signed: lists.some(x => x.signed)} } // JS: sort() compares UTF-16 units; build.py and LC_ALL=C sort compare code points function comparePaths(a, b) { const x = [...a], y = [...b]; for (let i = 0; i < x.length && i < y.length; i++) if (x[i] !== y[i]) return x[i].codePointAt(0) - y[i].codePointAt(0); return x.length - y.length } function formatList(items) { return [...items].sort((x, y) => comparePaths(x.path, y.path)).map(({hex, path}) => /[\\\n\r]/.test(path) ? `\\${hex} ${path.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/\r/g, "\\r")}\n` : `${hex} ${path}\n`).join("") } function matchFiles(entries, files) { const rest = new Map(); for (const r of entries) { const y = rest.get(r.path); if (!y || STRENGTH.indexOf(r.algorithm) > STRENGTH.indexOf(y.algorithm)) rest.set(r.path, r) } const bare = p => p.replace(/^(\.\/)+/, ""), nfc = p => bare(p).normalize("NFC"); const rel = x => x.folder && x.path.startsWith(x.folder + "/") ? x.path.slice(x.folder.length + 1) : x.path; const base = p => p.slice(p.lastIndexOf("/") + 1).normalize("NFC"); const pairs = [], ambiguous = [], left = new Set(files), dup = new Set(); for (const [how, f, g] of [ ["exact", r => r.path, x => [x.path]], ["dot-slash", r => bare(r.path), x => [bare(x.path)]], ["folder", r => bare(r.path), x => [bare(rel(x))]], ["nfc", r => nfc(r.path), x => [nfc(x.path), nfc(rel(x))]], ["basename", r => base(r.path), x => [base(x.path)]]]) { const by = new Map(), own = new Map(); for (const r of rest.values()) { const k = f(r); by.set(k, by.has(k) ? null : r) } for (const x of left) { const r = g(x).map(k => by.get(k)).find(Boolean); if (r) own.set(r, [...(own.get(r) ?? []), x]) } for (const [r, a] of own) { rest.delete(r.path); for (const x of a) left.delete(x); if (a.length === 1) pairs.push({entry: r, file: a[0], how}); else { ambiguous.push({entry: r, files: a}); for (const x of a) dup.add(x) } } } return {pairs, missing: [...rest.values()], unlisted: [...left].filter(x => !dup.has(x)), ambiguous} } function compareDigests(expectedHex, actualHex) { const x = expectedHex.toLowerCase(), y = actualHex.toLowerCase(), positions = []; if (x.length !== y.length) return {verdict: "differs", differing: Math.max(x.length, y.length), positions}; for (let i = 0; i < x.length; i++) if (x[i] !== y[i]) positions.push(i); const n = positions.length; return {verdict: n === 0 ? "match" : n <= 4 ? "typo" : "differs", differing: n, positions} } const otherAlgorithms = length => ({64: ["SHA3-256", "BLAKE3"], 96: ["SHA3-384"], 128: ["SHA3-512", "BLAKE2b"]})[length] ?? []; function invisibles(text) { let breaks = 0; while (text[text.length-1-breaks] === "\n") breaks++; const f = re => text.match(re)?.length ?? 0, bom = text.startsWith("\uFEFF"); const nbsp = f(/[\u00a0\u2007\u202f]/g); // Unicode: a Default_Ignorable_Code_Point draws as nothing; the direction marks among them have a count of their own const zero = f(/(?![\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069])\p{Default_Ignorable_Code_Point}/gu) - (bom ? 1 : 0); const marks = f(/[\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]/g); const lone = f(/[\ud800-\udbff](?![\udc00-\udfff])|(? s + "\n"], ["the line break at the end removed", s => s.replace(/\n$/, "")], ["Windows line breaks (CRLF)", s => s.replace(/\n/g, "\r\n")], ["Windows line breaks and one at the end", s => s.replace(/\n?$/, "\n").replace(/\n/g, "\r\n")], ["its accents composed (Unicode NFC)", s => s.normalize("NFC")], ["its accents decomposed (Unicode NFD)", s => s.normalize("NFD")], ["the spaces and line breaks at either end trimmed", s => s.trim()], ["a byte order mark at the start", s => "\uFEFF" + s]]; // inv: variants are told apart by their UTF-8, where a lone surrogate is U+FFFD function textVariants(text) { if (text.length > 2 ** 20) return []; const back = new TextDecoder("utf-8", {ignoreBOM: true}), seen = new Set([back.decode(utf8.encode(text))]); return VARIANTS.flatMap(([label, f]) => { const bytes = utf8.encode(f(text)), k = back.decode(bytes); if (seen.has(k)) return []; seen.add(k); return [{label, bytes}] }) } async function findVariant(text, {algorithm, hexes}) { for (const {label, bytes} of textVariants(text)) if (hexes.includes(await digest(algorithm, bytes))) return {label}; return null } const pause = () => globalThis.scheduler?.yield?.() ?? new Promise(f => setTimeout(f, 0)); async function hashSource(source, algorithms, {chunk = 2 ** 20, signal, onProgress} = {}) { const a = algorithms.map(createHash), {size} = source; const abort = () => new DOMException("Hashing was stopped.", "AbortError"); if (signal?.aborted) throw abort(); let done = 0, t = Date.now(), next = size ? source.read(0, Math.min(chunk, size)) : null; while (done < size) { const b = await next; if (signal?.aborted) throw abort(); b.length !== Math.min(chunk, size - done) && no("The file changed while it was read."); done += b.length; next = done < size ? source.read(done, Math.min(done + chunk, size)) : null; for (const x of a) x.update(b); onProgress?.(done, size); if (Date.now() - t >= 4) { await pause(); t = Date.now() } } if (!size) onProgress?.(0, 0); return Object.fromEntries(algorithms.map((k, i) => [k, toHex(a[i].digest())])) } const unsafe = p => !p || /[\0-\x1f\x7f-\x9f]/.test(p) || p.split("/").some(s => s === "" || s === "." || s === ".."); function sitePath(path) { unsafe(path) && no(`${displayName(path)} isn't a safe path.`); return "/" + path.split("/").map(encodeURIComponent).join("/") } function decodeManifest(bytes) { try { return new TextDecoder("utf-8", {fatal: true}).decode(bytes) } catch { no("manifest.txt isn't valid UTF-8.") } } // guard: an unreadable line refuses the list; skipping it would pass nonsense as zero files, all matching function parseManifest(text) { const o = [], seen = new Map(), a = text.split("\n"); for (let i = 0; i < a.length; i++) { const n = i + 1, at = {line: n}; if (a[i].trim() === "") continue; const [, digest, path] = /^([0-9a-f]+) [ *](.+)$/s.exec(a[i]) ?? []; digest || no(`Line ${n} isn't a checksum line.`, at); try { algorithmFor(digest) } catch (w) { no(`Line ${n}: ${w.message}`, at) } unsafe(path) && no(`Line ${n} names an unsafe path.`, at); seen.has(path) && no(`Lines ${seen.get(path)} and ${n} both name ${displayName(path)}.`, at); seen.set(path, n); o.push({digest, path, line: n}) > MOST && no(`It lists more than ${num(MOST)} files.`, at) } o.length || no("The manifest lists no files."); return o } async function checkEntries(entries, load, {concurrency = 6, onProgress} = {}) { const o = []; let next = 0, done = 0; await Promise.all(Array.from({length: Math.min(concurrency, entries.length)}, async () => { while (next < entries.length) { const i = next++, r = entries[i]; try { const b = await load(r.path), actual = await digest(algorithmFor(r.digest), b), ok = actual === r.digest; o[i] = {...r, actual, size: b.byteLength, ok, status: ok ? "ok" : "changed"} } catch (w) { o[i] = {...r, actual: null, ok: false, status: w?.status === 404 ? "missing" : "unreadable", error: String(w)} } onProgress?.(++done, entries.length) } })); return o } function parseColophon(html) { const m = /
  • digest ([0-9a-f]{64})<\/code><\/span><\/li>/.exec(html); const releases = [...html.matchAll(/