diff --git a/scripts/lib/mcode-tools-artifact.mjs b/scripts/lib/mcode-tools-artifact.mjs index 98868d243..0c4fe695a 100644 --- a/scripts/lib/mcode-tools-artifact.mjs +++ b/scripts/lib/mcode-tools-artifact.mjs @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { Readable } from 'node:stream'; +import { setTimeout as sleep } from 'node:timers/promises'; import { Parser } from 'tar'; export const MCODE_TOOLS_ARTIFACT = Object.freeze({ @@ -51,15 +52,52 @@ export async function extractMcodeToolsArtifact(bytes) { return { cli, manifest: manifestBytes, notices }; } +const transientNetworkCodes = new Set([ + 'ECONNRESET', 'ECONNREFUSED', 'EPIPE', 'ETIMEDOUT', 'EAI_AGAIN', 'ENETUNREACH', + 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_SOCKET', +]); +const transientHttpStatuses = new Set([408, 429, 500, 502, 503, 504]); + +export async function downloadMcodeToolsArtifact(fetchImpl = fetch, { wait = sleep, warn = console.warn } = {}) { + const attempts = 3; + for (let attempt = 1; attempt <= attempts; attempt++) { + let retryAfterMs = 0; + try { + // A new deadline covers both connection setup and the complete response body. + const response = await fetchImpl(MCODE_TOOLS_ARTIFACT.url, { signal: AbortSignal.timeout(120_000) }); + if (!response.ok) { + const retryAfter = response.headers.get('retry-after'); + if (retryAfter !== null) { + const milliseconds = /^\d+$/.test(retryAfter) ? Number(retryAfter) * 1000 : Date.parse(retryAfter) - Date.now(); + if (!Number.isNaN(milliseconds)) retryAfterMs = Math.min(30_000, Math.max(0, milliseconds)); + } + await response.body?.cancel().catch(() => {}); + const error = new Error(`Cannot download public mcode-tools artifact: HTTP ${response.status}`); + error.status = response.status; + throw error; + } + return Buffer.from(await response.arrayBuffer()); + } catch (error) { + const code = error.cause?.code ?? error.code; + const retryable = transientHttpStatuses.has(error.status) || + transientNetworkCodes.has(code) || error.name === 'TimeoutError'; + if (!retryable) throw error; + if (attempt === attempts) + throw new Error(`Cannot download public mcode-tools artifact after ${attempts} attempts`, { cause: error }); + const delayMs = Math.max(1000 * 2 ** (attempt - 1), retryAfterMs); + warn(`[mcode-tools] Download attempt ${attempt}/${attempts} failed (${code ?? error.message}); retrying in ${delayMs} ms.`); + await wait(delayMs); + } + } +} + export async function copyMcodeToolsArtifact(root, outdir, fetchImpl = fetch) { const cache = path.join(root, '.cache', 'artifacts', 'code-0.3.11.tgz'); let bytes; try { bytes = await readFile(cache); } catch (error) { if (error.code !== 'ENOENT') throw error; - const response = await fetchImpl(MCODE_TOOLS_ARTIFACT.url, { signal: AbortSignal.timeout(120_000) }); - if (!response.ok) throw new Error(`Cannot download public mcode-tools artifact: HTTP ${response.status}`); - bytes = Buffer.from(await response.arrayBuffer()); + bytes = await downloadMcodeToolsArtifact(fetchImpl); } const artifact = await extractMcodeToolsArtifact(bytes); await mkdir(path.dirname(cache), { recursive: true }); diff --git a/test/source-sync.test.mjs b/test/source-sync.test.mjs index 8a92c52f8..2833c05fc 100644 --- a/test/source-sync.test.mjs +++ b/test/source-sync.test.mjs @@ -19,6 +19,93 @@ import { releaseManifest } from '../scripts/package-cli-release.mjs'; import { validateReleaseReports } from '../scripts/publish-cli-release.mjs'; import { compareVersions, releaseCli } from '../scripts/release-cli.mjs'; import { compareRuns, validateRun, validateRequest, validateToolOutput, median, selectScenarios } from '../scripts/perf/report.mjs'; +import { copyMcodeToolsArtifact, downloadMcodeToolsArtifact, MCODE_TOOLS_ARTIFACT } from '../scripts/lib/mcode-tools-artifact.mjs'; + +test('artifact download recovers from TLS reset and interrupted response bodies', async () => { + const reset = new TypeError('fetch failed', { cause: Object.assign(new Error('connection reset'), { code: 'ECONNRESET' }) }); + const interrupted = new TypeError('terminated', { cause: Object.assign(new Error('socket closed'), { code: 'UND_ERR_SOCKET' }) }); + const signals = [], delays = [], warnings = []; + const result = await downloadMcodeToolsArtifact(async (url, { signal }) => { + assert.equal(url, MCODE_TOOLS_ARTIFACT.url); + assert.ok(signal instanceof AbortSignal); + signals.push(signal); + if (signals.length === 1) throw reset; + if (signals.length === 2) return new Response(new ReadableStream({ + start(controller) { controller.enqueue(new Uint8Array([1, 2])); }, + pull(controller) { controller.error(interrupted); }, + })); + return new Response('complete archive'); + }, { wait: async ms => delays.push(ms), warn: message => warnings.push(message) }); + assert.equal(result.toString(), 'complete archive'); + assert.equal(new Set(signals).size, 3); + assert.deepEqual(delays, [1000, 2000]); + assert.equal(warnings.length, 2); + assert.match(warnings[0], /1\/3.*ECONNRESET/); + assert.match(warnings[1], /2\/3.*UND_ERR_SOCKET/); +}); + +test('artifact download retries temporary HTTP failures and releases rejected bodies', async () => { + for (const status of [408, 429, 500, 502, 503, 504]) { + let calls = 0, cancelled = false; + const delays = []; + const result = await downloadMcodeToolsArtifact(async () => { + if (++calls > 1) return new Response('ok'); + return new Response(new ReadableStream({ cancel() { cancelled = true; } }), { status }); + }, { wait: async ms => delays.push(ms), warn() {} }); + assert.equal(result.toString(), 'ok'); + assert.equal(calls, 2); + assert.equal(cancelled, true); + assert.deepEqual(delays, [1000]); + } +}); + +test('artifact download bounds Retry-After delays and rejects permanent failures immediately', async () => { + for (const [header, expectedDelay] of [['5', 5000], ['999999', 30000], ['invalid', 1000], ['0', 1000]]) { + let calls = 0; + const delays = []; + await downloadMcodeToolsArtifact(async () => ++calls === 1 + ? new Response(null, { status: 429, headers: { 'Retry-After': header } }) : new Response('ok'), + { wait: async ms => delays.push(ms), warn() {} }); + assert.deepEqual(delays, [expectedDelay]); + } + const certificateError = new TypeError('fetch failed', { cause: Object.assign(new Error('certificate expired'), { code: 'CERT_HAS_EXPIRED' }) }); + for (const failure of [new Response(null, { status: 403 }), new Response(null, { status: 404 }), certificateError]) { + let calls = 0; + await assert.rejects(downloadMcodeToolsArtifact(async () => { + calls++; + if (failure instanceof Error) throw failure; + return failure; + }, { wait: async () => assert.fail('permanent failures must not wait'), warn: () => assert.fail('permanent failures must not retry') })); + assert.equal(calls, 1); + } +}); + +test('artifact download stops after three timeouts and preserves the final cause', async () => { + const timeout = new DOMException('The operation timed out', 'TimeoutError'); + let calls = 0; + const delays = []; + await assert.rejects(downloadMcodeToolsArtifact(async () => { calls++; throw timeout; }, + { wait: async ms => delays.push(ms), warn() {} }), error => { + assert.match(error.message, /after 3 attempts/); + assert.equal(error.cause, timeout); + return true; + }); + assert.equal(calls, 3); + assert.deepEqual(delays, [1000, 2000]); +}); + +test('artifact download never retries or caches an archive that fails integrity', async t => { + const root = mkdtempSync(path.join(tmpdir(), 'mcode-artifact-download-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + let calls = 0; + await assert.rejects(copyMcodeToolsArtifact(root, path.join(root, 'dist'), async () => { + calls++; + return new Response('corrupt archive'); + }), /integrity mismatch/); + assert.equal(calls, 1); + assert.equal(existsSync(path.join(root, '.cache', 'artifacts', 'code-0.3.11.tgz')), false); + assert.equal(existsSync(path.join(root, 'dist')), false); +}); test('performance defaults to the 100-round suite; long history requires explicit selection', () => { const config = JSON.parse(readFileSync(new URL('../scripts/perf/config.json', import.meta.url), 'utf8'));