|
| 1 | +#!/usr/bin/env node |
| 2 | +/** |
| 3 | + * Run the backend suite as SEVERAL `node --test` invocations instead of one. |
| 4 | + * |
| 5 | + * ═════════════════════════════════════════════════════════════════════════════ |
| 6 | + * WHY THIS EXISTS |
| 7 | + * ═════════════════════════════════════════════════════════════════════════════ |
| 8 | + * `node --test` runs each test file in a child process and streams the results |
| 9 | + * back over an IPC channel that the PARENT deserializes. On every Node 22 there |
| 10 | + * is a bug in that parent-side reader: the per-message size is decoded from four |
| 11 | + * bytes with `<<`, which is a SIGNED operation in JavaScript, so a size whose |
| 12 | + * top byte has the high bit set comes out negative and the next slice is taken |
| 13 | + * from the wrong place. What surfaces is: |
| 14 | + * |
| 15 | + * not ok N - <an arbitrary file> |
| 16 | + * failureType: 'uncaughtException' |
| 17 | + * error: 'Unable to deserialize cloned data due to invalid or unsupported version.' |
| 18 | + * stack: #processRawBuffer (node:internal/test_runner/runner.js) |
| 19 | + * |
| 20 | + * — a failure with no assertion in it, blaming a file that did nothing wrong, |
| 21 | + * and taking the rest of that file's tests down with it (the reported test count |
| 22 | + * DROPS, which is how you tell it from a real red). |
| 23 | + * |
| 24 | + * Upstream fixed it in `nodejs/node#64706` by making that decode unsigned. The |
| 25 | + * fix is in **v24.20.0 and v26.7.0**. It is in **no** release of Node 22 — |
| 26 | + * 22.23.2 is the newest and does not carry it, and the PR has no v22 backport |
| 27 | + * label. This repo's runtime image is `node:22-alpine`, so CI runs 22 on |
| 28 | + * purpose: testing on a Node the container does not ship would be a worse trade |
| 29 | + * than the flake. |
| 30 | + * |
| 31 | + * ═════════════════════════════════════════════════════════════════════════════ |
| 32 | + * WHAT SHARDING ACTUALLY BUYS, STATED HONESTLY |
| 33 | + * ═════════════════════════════════════════════════════════════════════════════ |
| 34 | + * Each shard is its own parent process reading its own IPC stream, so no single |
| 35 | + * parent decodes the whole suite's messages any more — roughly a quarter each. |
| 36 | + * The bug is per-message and probabilistic, so this REDUCES exposure roughly in |
| 37 | + * proportion. **It does not remove the mechanism, and this file does not claim |
| 38 | + * to have fixed anything.** The cure is Node ≥ 24.20 (or a v22 backport that |
| 39 | + * does not exist yet); until one of those, this is a smaller target. |
| 40 | + * |
| 41 | + * There is deliberately NO RETRY here. A retry that turns a genuine red into a |
| 42 | + * green is worse than the flake it hides, and a retry loud enough to be safe |
| 43 | + * (named in the summary, second failure still fails the job) is a bigger change |
| 44 | + * than this one — worth building only if sharding proves not to be enough. |
| 45 | + * |
| 46 | + * ═════════════════════════════════════════════════════════════════════════════ |
| 47 | + * WHY `--test-shard` AND NOT A HAND-WRITTEN FILE LIST |
| 48 | + * ═════════════════════════════════════════════════════════════════════════════ |
| 49 | + * Node does its own discovery and its own partitioning. A chunker that globbed |
| 50 | + * for test files itself would have to reimplement those rules — and the failure |
| 51 | + * mode of getting them slightly wrong is a directory that silently stops being |
| 52 | + * tested, which is far worse than an occasional red build. The guards below |
| 53 | + * exist for the one way this script itself could drop tests: a wrong index. |
| 54 | + * |
| 55 | + * ═════════════════════════════════════════════════════════════════════════════ |
| 56 | + * AND WHY IT IS NOT CALLED `test-shards.mjs` |
| 57 | + * ═════════════════════════════════════════════════════════════════════════════ |
| 58 | + * It was, for about ten minutes. A leading `test-` is one of Node's default |
| 59 | + * test-file patterns, so `node --test` discovered this runner as a TEST FILE, ran it |
| 60 | + * inside a shard, and that nested run found no files and exited 1 — a red build |
| 61 | + * caused by the thing meant to make builds less red. It also inflated the count |
| 62 | + * by one, which is what gave it away. |
| 63 | + * |
| 64 | + * Any name matching `test-*`, `*-test`, `*_test`, `*.test` or anything under a |
| 65 | + * `test/` directory will be executed rather than merely run. This is the same |
| 66 | + * discovery subtlety that makes hand-rolling the file list a bad idea. |
| 67 | + * |
| 68 | + * Usage: node scripts/shard-runner.mjs (from backend/) |
| 69 | + * TEST_SHARDS=6 node scripts/shard-runner.mjs |
| 70 | + */ |
| 71 | + |
| 72 | +import { spawnSync } from 'node:child_process'; |
| 73 | +import { appendFileSync } from 'node:fs'; |
| 74 | + |
| 75 | +/** |
| 76 | + * Four is a balance, not a magic number: enough that no parent carries most of |
| 77 | + * the stream, few enough that the per-process start-up cost stays small against |
| 78 | + * a ~60s suite. Overridable so a bisect can try other values without a commit. |
| 79 | + */ |
| 80 | +const TOTAL = Number.parseInt(process.env.TEST_SHARDS ?? '4', 10); |
| 81 | +if (!Number.isInteger(TOTAL) || TOTAL < 1) { |
| 82 | + console.error(`[shard-runner] TEST_SHARDS must be a positive integer, got ${process.env.TEST_SHARDS}`); |
| 83 | + process.exit(2); |
| 84 | +} |
| 85 | + |
| 86 | +/** `# tests 2032` and friends, off the end of a shard's own summary. */ |
| 87 | +function counters(output) { |
| 88 | + const read = (name) => { |
| 89 | + const m = output.match(new RegExp(`^# ${name} (\\d+)$`, 'm')); |
| 90 | + return m ? Number(m[1]) : null; |
| 91 | + }; |
| 92 | + return { tests: read('tests'), pass: read('pass'), fail: read('fail'), skipped: read('skipped') }; |
| 93 | +} |
| 94 | + |
| 95 | +const started = Date.now(); |
| 96 | +/** @type {{index: number, code: number|null, tests: number|null, pass: number|null, fail: number|null, skipped: number|null}[]} */ |
| 97 | +const shards = []; |
| 98 | + |
| 99 | +for (let index = 1; index <= TOTAL; index++) { |
| 100 | + console.log(`\n═══ shard ${index}/${TOTAL} ═══`); |
| 101 | + const run = spawnSync( |
| 102 | + process.execPath, |
| 103 | + ['--test', '--test-concurrency=1', `--test-shard=${index}/${TOTAL}`], |
| 104 | + { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 } |
| 105 | + ); |
| 106 | + // Streamed after the fact rather than inherited, because the counters have to |
| 107 | + // be read back out of it. The whole output is printed either way, so a real |
| 108 | + // failure is as legible in the log as it was before. |
| 109 | + process.stdout.write(run.stdout ?? ''); |
| 110 | + if (run.stderr) process.stderr.write(run.stderr); |
| 111 | + shards.push({ index, code: run.status, ...counters(run.stdout ?? '') }); |
| 112 | +} |
| 113 | + |
| 114 | +const sum = (key) => shards.reduce((a, s) => a + (s[key] ?? 0), 0); |
| 115 | +const totals = { |
| 116 | + tests: sum('tests'), |
| 117 | + pass: sum('pass'), |
| 118 | + fail: sum('fail'), |
| 119 | + skipped: sum('skipped'), |
| 120 | +}; |
| 121 | + |
| 122 | +const lines = [ |
| 123 | + `backend suite, ${TOTAL} shards, ${((Date.now() - started) / 1000).toFixed(0)}s`, |
| 124 | + ...shards.map( |
| 125 | + (s) => |
| 126 | + ` shard ${s.index}/${TOTAL}: exit ${s.code} · ${s.tests ?? '?'} tests · ` + |
| 127 | + `${s.pass ?? '?'} pass · ${s.fail ?? '?'} fail · ${s.skipped ?? '?'} skipped` |
| 128 | + ), |
| 129 | + ` TOTAL: ${totals.tests} tests · ${totals.pass} pass · ${totals.fail} fail · ${totals.skipped} skipped`, |
| 130 | +]; |
| 131 | +console.log(`\n${lines.join('\n')}`); |
| 132 | + |
| 133 | +/* |
| 134 | + * THE SUMMARY IS WRITTEN WHERE A PERSON WILL SEE IT WITHOUT OPENING THE LOG. |
| 135 | + * |
| 136 | + * The whole point of splitting the run is that the totals are now assembled by |
| 137 | + * this script rather than printed by Node — so if the assembling is wrong, it |
| 138 | + * has to be wrong somewhere visible. |
| 139 | + */ |
| 140 | +if (process.env.GITHUB_STEP_SUMMARY) { |
| 141 | + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### Backend tests\n\n\`\`\`\n${lines.join('\n')}\n\`\`\`\n`); |
| 142 | +} |
| 143 | + |
| 144 | +const problems = []; |
| 145 | +for (const s of shards) { |
| 146 | + if (s.code !== 0) problems.push(`shard ${s.index}/${TOTAL} exited ${s.code}`); |
| 147 | + /* |
| 148 | + * A SHARD THAT RAN NOTHING IS THE ONE WAY THIS SCRIPT COULD SILENTLY DROP |
| 149 | + * TESTS — a wrong index, or an off-by-one in the loop, hands Node a shard it |
| 150 | + * has no files for and it exits 0 with an empty run. Node's own discovery |
| 151 | + * cannot produce an empty shard for this repo: there are ~111 test files and |
| 152 | + * four shards. |
| 153 | + */ |
| 154 | + if (!s.tests) problems.push(`shard ${s.index}/${TOTAL} reported no tests at all`); |
| 155 | +} |
| 156 | +if (totals.fail > 0) problems.push(`${totals.fail} failing test(s)`); |
| 157 | + |
| 158 | +if (problems.length > 0) { |
| 159 | + console.error(`\n[shard-runner] FAILED: ${problems.join('; ')}`); |
| 160 | + process.exit(1); |
| 161 | +} |
| 162 | +console.log(`[shard-runner] ${TOTAL} shards, all green`); |
0 commit comments