|
| 1 | +/** |
| 2 | + * Offline Unicode 12.0.0 IDNA2008 table generator. |
| 3 | + * Run: node --experimental-strip-types scripts/generate-idna.ts INPUT_DIR OUTPUT_DIR [--check] |
| 4 | + * Supply the named pinned source files below; this script never fetches data. |
| 5 | + * OUTPUT_DIR is packages/fedify/src/delivery-suspension in the checkout. |
| 6 | + */ |
| 7 | +import { createHash } from "node:crypto"; |
| 8 | +import { readFileSync, writeFileSync } from "node:fs"; |
| 9 | + |
| 10 | +const sources = { |
| 11 | + "iana12.csv": [ |
| 12 | + "https://www.iana.org/assignments/idna-tables-12.0.0/idna-tables-properties.csv", |
| 13 | + "160b72e33a377a84253393c572dc338879f39b93beccbb9cb9baebbf4d6dd743", |
| 14 | + ], |
| 15 | + "UnicodeData12.txt": [ |
| 16 | + "https://www.unicode.org/Public/12.0.0/ucd/UnicodeData.txt", |
| 17 | + "57913fdb1c1271fa9fd51dea08d49370f33d5bfda6bbaba348f44d90029ea11d", |
| 18 | + ], |
| 19 | + "DerivedJoiningType12.txt": [ |
| 20 | + "https://www.unicode.org/Public/12.0.0/ucd/extracted/DerivedJoiningType.txt", |
| 21 | + "2a92a39d6713b8cc9a6d4eb808efaf93c52a4fcc01c149fec9892113838dae8b", |
| 22 | + ], |
| 23 | + "Scripts12.txt": [ |
| 24 | + "https://www.unicode.org/Public/12.0.0/ucd/Scripts.txt", |
| 25 | + "bd327cbb8bffd2e4b40b1e9842c9a7d641d1c4e91c34a41020cf174f3a1c86a1", |
| 26 | + ], |
| 27 | + "UnicodeLicense.txt": [ |
| 28 | + "https://www.unicode.org/license.txt", |
| 29 | + "e7a93b009565cfce55919a381437ac4db883e9da2126fa28b91d12732bc53d96", |
| 30 | + ], |
| 31 | +} as const; |
| 32 | +const [input, output, mode] = process.argv.slice(2); |
| 33 | +if (!input || !output || (mode != null && mode !== "--check")) { |
| 34 | + throw new Error("Usage: generate-idna.ts INPUT_DIR OUTPUT_DIR [--check]"); |
| 35 | +} |
| 36 | +function hash(text: string): string { |
| 37 | + return createHash("sha256").update(text).digest("hex"); |
| 38 | +} |
| 39 | +const files: Record<string, string> = {}; |
| 40 | +for (const [name, [, digest]] of Object.entries(sources)) { |
| 41 | + const bytes = readFileSync(`${input}/${name}`); |
| 42 | + if (createHash("sha256").update(bytes).digest("hex") !== digest) { |
| 43 | + throw new Error(`Source hash mismatch: ${name}`); |
| 44 | + } |
| 45 | + files[name] = bytes.toString("utf8"); |
| 46 | +} |
| 47 | +const bidiNames = [ |
| 48 | + "", |
| 49 | + "L", |
| 50 | + "R", |
| 51 | + "AL", |
| 52 | + "AN", |
| 53 | + "EN", |
| 54 | + "ES", |
| 55 | + "CS", |
| 56 | + "ET", |
| 57 | + "ON", |
| 58 | + "BN", |
| 59 | + "NSM", |
| 60 | +]; |
| 61 | +const joiningNames = ["", "L", "R", "D", "T"]; |
| 62 | +const scriptNames = ["", "Greek", "Hebrew", "Han", "Hiragana", "Katakana"]; |
| 63 | +const arrays = Object.fromEntries( |
| 64 | + ["validity", "assigned", "mark", "virama", "bidi", "joining", "script"].map( |
| 65 | + (key) => [key, new Uint8Array(0x110000)], |
| 66 | + ), |
| 67 | +); |
| 68 | +function fill(key: string, start: number, end: number, value: number): void { |
| 69 | + if (value > 0) arrays[key].fill(value, start, end + 1); |
| 70 | +} |
| 71 | +function range(value: string): [number, number] { |
| 72 | + const [start, end = start] = value |
| 73 | + .split(/\.\.|-/) |
| 74 | + .map((part) => parseInt(part, 16)); |
| 75 | + if (!(start >= 0 && end >= start && end <= 0x10ffff)) |
| 76 | + throw new Error("Invalid source range"); |
| 77 | + return [start, end]; |
| 78 | +} |
| 79 | +for (const line of files["iana12.csv"].split(/\r?\n/).slice(1)) { |
| 80 | + const [points, property] = line.split(","); |
| 81 | + const value = ["", "PVALID", "CONTEXTJ", "CONTEXTO"].indexOf(property); |
| 82 | + if (value > 0) fill("validity", ...range(points), value); |
| 83 | +} |
| 84 | +let first: string[] | undefined; |
| 85 | +for (const line of files["UnicodeData12.txt"].trim().split(/\r?\n/)) { |
| 86 | + const fields = line.split(";"); |
| 87 | + if (fields[1].endsWith(", First>")) { |
| 88 | + if (first) throw new Error("Nested First range"); |
| 89 | + first = fields; |
| 90 | + continue; |
| 91 | + } |
| 92 | + const start = parseInt((first ?? fields)[0], 16); |
| 93 | + const end = parseInt(fields[0], 16); |
| 94 | + if (first && !fields[1].endsWith(", Last>")) |
| 95 | + throw new Error("Unclosed First range"); |
| 96 | + if (!first && fields[1].endsWith(", Last>")) |
| 97 | + throw new Error("Unexpected Last range"); |
| 98 | + if (first && first.slice(2).join(";") !== fields.slice(2).join(";")) |
| 99 | + throw new Error("Inconsistent First/Last properties"); |
| 100 | + fill("assigned", start, end, 1); |
| 101 | + fill("mark", start, end, Number(fields[2].startsWith("M"))); |
| 102 | + fill("virama", start, end, Number(fields[3] === "9")); |
| 103 | + fill("bidi", start, end, bidiNames.indexOf(fields[4])); |
| 104 | + first = undefined; |
| 105 | +} |
| 106 | +if (first) throw new Error("Unclosed First range"); |
| 107 | +for (const [file, key, names] of [ |
| 108 | + ["DerivedJoiningType12.txt", "joining", joiningNames], |
| 109 | + ["Scripts12.txt", "script", scriptNames], |
| 110 | +] as const) { |
| 111 | + for (const line of files[file].split(/\r?\n/)) { |
| 112 | + const match = /^\s*([0-9A-F.]+)\s*;\s*(\w+)/.exec(line); |
| 113 | + if (match) fill(key, ...range(match[1]), names.indexOf(match[2])); |
| 114 | + } |
| 115 | +} |
| 116 | +// Every array is an ordered sequence of inclusive [first, last, property] triples. |
| 117 | +const tables: Record<string, number[]> = {}; |
| 118 | +for (const [key, values] of Object.entries(arrays)) { |
| 119 | + const ranges: number[] = []; |
| 120 | + for (let i = 0; i < values.length;) { |
| 121 | + const start = i, |
| 122 | + value = values[i++]; |
| 123 | + while (i < values.length && values[i] === value) i++; |
| 124 | + if (value > 0) ranges.push(start, i - 1, value); |
| 125 | + } |
| 126 | + tables[key] = ranges; |
| 127 | +} |
| 128 | +const digest = hash(JSON.stringify(tables)); |
| 129 | +const sourceComments = Object.entries(sources) |
| 130 | + .map(([name, [url, digest]]) => `// ${name}: ${url}\n// SHA-256: ${digest}`) |
| 131 | + .join("\n"); |
| 132 | +const source = `/*! @license\n${files["UnicodeLicense.txt"].trimEnd()}\n*/\n// Generated by scripts/generate-idna.ts; do not edit.\n// Unicode 12.0.0. See UNICODE-LICENSE.txt in this directory.\n${sourceComments}\nexport const IDNA_UNICODE_VERSION = "12.0.0";\nexport const IDNA_DATA_SHA256 =\n "${digest}";\n// prettier-ignore\nexport const IDNA_TABLES: Readonly<Record<"validity" | "assigned" | "mark" | "virama" | "bidi" | "joining" | "script", readonly number[]>> = ${JSON.stringify(tables, null, 0)};\n`; |
| 133 | +for (const [name, content] of [ |
| 134 | + ["idna-data.ts", source], |
| 135 | + ["UNICODE-LICENSE.txt", files["UnicodeLicense.txt"]], |
| 136 | +]) { |
| 137 | + const path = `${output}/${name}`; |
| 138 | + if (mode === "--check") { |
| 139 | + if (readFileSync(path, "utf8") !== content) |
| 140 | + throw new Error(`Generated file differs: ${path}`); |
| 141 | + } else writeFileSync(path, content); |
| 142 | +} |
| 143 | +process.stdout.write( |
| 144 | + `Unicode 12.0.0 IDNA tables ${mode === "--check" ? "verified" : "generated"}: ${digest}\n`, |
| 145 | +); |
0 commit comments