From 2bf263f9a44667737112c0fc6ee1d35eb99a2536 Mon Sep 17 00:00:00 2001 From: jdalton Date: Wed, 22 Apr 2026 19:58:19 -0400 Subject: [PATCH 1/3] feat(build): port scripts/build.mts to shared build-pipeline orchestrator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth copy of the shared build-pipeline system (socket-btm + ultrathink + socket-tui + sdxgen all use it). API surface is identical across repos — manifest-of-stages, --force/--clean/--clean-stage/--from-stage/ --cache-key CLI, checkpoint JSONs under build//. What landed - packages/build-infra/lib: adds checkpoint-manager, constants, external-tools-schema, platform-mappings, version-helpers, build-pipeline alongside the existing esbuild/platform-targets/ github-releases helpers. Coexists — no touching the pre-existing files. - constants.mts: socket-cli's own checkpoint chain (CLI → SEA → FINALIZED). No wasm verbs — socket-cli consumes pre-built wasm + node binaries from socket-btm; same orchestrator drives a pure JS build, the module name 'build-pipeline' reflects that. - build-pipeline.mts: add `skip?: (ctx) => boolean` dynamic skip predicate. skipInDev stays for fleet parity; skip is for dynamic conditions like socket-cli's SEA stage (only runs on --force/--prod). - scripts/build.mts: replace runSmartBuild's procedural loop with a runPipelineCli call. Existing buildPackage/buildCurrentPlatformSea helpers are wrapped as stage workers; the existing BUILD_PACKAGES signature system still runs inside buildPackage's body, complementing the orchestrator's own cache-hash layer (the two aren't fighting — one works on file-glob inputs, the other on content hashes of sourcePaths + platform metadata). - Dispatch paths unchanged: --platforms / --targets / --target still route to runParallelBuilds / runSequentialBuilds / runTargetedBuild (the orchestrator only replaces the default smart-build path). - Platform pinned to 'universal' via resolvePlatformArch so the cache key stays stable across runner OSes (bundled CLI JS is universal). - Checkpoints at build//checkpoints/ matching socket-tui + sdxgen. Bump @socketsecurity/lib catalog pin 5.21 → 5.24. 5.21's /errors subpath shipped CJS without named-export interop, so `import { errorMessage } from '@socketsecurity/lib/errors'` failed under Node's ESM-to-CJS resolver. 5.24 ships the interop (already in use by socket-tui, ultrathink, socket-sdxgen). Also: add @sinclair/typebox to catalog (needed by external-tools-schema.mts). Verified locally: `pnpm build` completes in 16s, CLI Package builds, SEA correctly skipped (skip predicate), FINALIZED writes checkpoint. Cached rerun: 0.0s. --help / --target / --platforms paths unchanged. --- packages/build-infra/lib/build-pipeline.mts | 482 ++++++++++++++++++ .../build-infra/lib/checkpoint-manager.mts | 252 +++++++++ packages/build-infra/lib/constants.mts | 109 ++++ .../lib/external-tools-schema.json | 54 ++ .../build-infra/lib/external-tools-schema.mts | 124 +++++ .../build-infra/lib/platform-mappings.mts | 234 +++++++++ packages/build-infra/lib/version-helpers.mts | 56 ++ packages/build-infra/package.json | 9 +- pnpm-lock.yaml | 44 +- pnpm-workspace.yaml | 3 +- scripts/build.mts | 160 +++--- 11 files changed, 1428 insertions(+), 99 deletions(-) create mode 100644 packages/build-infra/lib/build-pipeline.mts create mode 100644 packages/build-infra/lib/checkpoint-manager.mts create mode 100644 packages/build-infra/lib/constants.mts create mode 100644 packages/build-infra/lib/external-tools-schema.json create mode 100644 packages/build-infra/lib/external-tools-schema.mts create mode 100644 packages/build-infra/lib/platform-mappings.mts create mode 100644 packages/build-infra/lib/version-helpers.mts diff --git a/packages/build-infra/lib/build-pipeline.mts b/packages/build-infra/lib/build-pipeline.mts new file mode 100644 index 000000000..96caed3df --- /dev/null +++ b/packages/build-infra/lib/build-pipeline.mts @@ -0,0 +1,482 @@ +/** + * WASM build pipeline orchestrator. + * + * Declarative orchestrator for wasm-shipping packages. Given a manifest of + * ordered stages, it drives the canonical sequence: + * clone source → configure → compile → release → (optimize) → sync → finalize + * + * The orchestrator owns every moving part that today lives in each package's + * 340-line build.mts: + * + * - Build mode + platform-arch detection (uses centralized helpers). + * - Loading external-tools.json + package.json `sources` metadata. + * - Deriving a unified cache key from: node version, platform, arch, build + * mode, pinned tool versions, and source refs. Tool bump or source SHA bump + * invalidates the cache automatically — no hand-wired busting. + * - Per-stage shouldRun() / createCheckpoint() wrapping. Stages become pure + * work functions; they do not implement skip-if-cached themselves. + * - Common CLI flags: --prod / --dev / --force / --clean / + * --clean-stage= / --from-stage= / --cache-key. + * + * A stage is `(ctx, params) => Promise`. `ctx` carries derived values + * shared by every stage (paths, mode, logger, tool versions, source meta). + * `params` holds stage-local overrides from the manifest. + * + * @module build-infra/lib/build-pipeline + */ + +import { createHash } from 'node:crypto' +import { existsSync, promises as fs, readFileSync } from 'node:fs' +import path from 'node:path' +import process from 'node:process' + +import { getDefaultLogger } from '@socketsecurity/lib/logger' + +import { errorMessage } from '@socketsecurity/lib/errors' + +import { + cleanCheckpoint, + createCheckpoint, + shouldRun, +} from './checkpoint-manager.mts' +import { getBuildMode, validateCheckpointChain } from './constants.mts' +import { validateExternalTools } from './external-tools-schema.mts' +import { getCurrentPlatformArch } from './platform-mappings.mts' +import { getNodeVersion } from './version-helpers.mts' + +const logger = getDefaultLogger() + +/** + * @typedef {object} StageResult + * @property {() => Promise | void} [smokeTest] + * Post-run validation. Runs before the checkpoint is committed. + * @property {string} [artifactPath] + * Absolute path archived into the checkpoint tarball. + * @property {string} [binaryPath] + * Relative path (from buildDir) to a binary to codesign on macOS. + * @property {string | number} [binarySize] + * Optional size metadata surfaced in checkpoint data. + */ + +/** + * @typedef {object} PipelineStage + * @property {string} name - Checkpoint name (must appear in CHECKPOINTS). + * @property {(ctx: PipelineContext, params?: object) => Promise} run + * Stage worker. Receives the shared context and optional per-stage params. + * Should perform the build work only — no shouldRun / createCheckpoint calls. + * Return a StageResult to configure the checkpoint (smoke test + artifact). + * @property {string[]} [sourcePaths] + * Extra file paths whose content contributes to this stage's cache hash. The + * orchestrator always includes package-wide inputs (external-tools.json, + * package.json); list stage-specific inputs here (e.g. an optimization + * flags module). + * @property {boolean} [skipInDev] + * Skip this stage entirely when buildMode === 'dev' (e.g. wasm-optimized). + * @property {(ctx: PipelineContext) => boolean} [skip] + * Dynamic skip predicate. Runs before shouldRun(). When it returns true, + * the stage is skipped without being recorded as cached. Use when the + * skip condition depends on runtime context beyond buildMode (e.g. + * socket-cli's SEA stage, which only runs when --force is present). + * @property {boolean} [shared] + * Checkpoint lives at the shared build dir instead of per-platform (e.g. + * source-cloned, which is platform-agnostic). + */ + +/** + * @typedef {object} PipelineContext + * @property {string} packageRoot - Package root (absolute). + * @property {string} packageName - Friendly name used in logs. + * @property {string} buildMode - 'dev' or 'prod'. + * @property {string} platformArch - Canonical platform-arch string. + * @property {string} nodeVersion - Node version running the build. + * @property {boolean} forceRebuild - Global --force flag. + * @property {Record} toolVersions - Map of tool name -> pinned version. + * @property {Record} sources - Contents of package.json `sources`. + * @property {object} paths - Result of the package's getBuildPaths(mode, platformArch). + * @property {object} sharedPaths - Result of the package's getSharedBuildPaths(), if any. + * @property {string} cacheKey - Unified cache key for GH Actions. + * @property {typeof logger} logger + */ + +/** + * @typedef {object} RunPipelineOptions + * @property {string} packageRoot - Absolute path to the package directory. + * @property {string} packageName - Short name used in logs (e.g. 'yoga'). + * @property {PipelineStage[]} stages - Stages in execution order. + * @property {(mode: string, platformArch: string) => object} getBuildPaths + * Package's path resolver for mode + platformArch. + * @property {() => object} [getSharedBuildPaths] + * Optional shared-path resolver (for source-cloned tarballs). + * @property {() => Promise} [preflight] + * Optional pre-build check (tool probing, disk space). Runs once before + * the first stage. Throws to abort the build. + * @property {(paths: object) => string[]} [getOutputFiles] + * Returns absolute paths to the artifacts the build is expected to emit. + * Missing files trigger a full-checkpoint clean to force a rebuild. + * @property {string[]} [extraCacheInputs] + * Extra file paths whose content is mixed into the cache key. Package-wide + * inputs (external-tools.json + package.json) are already included. + * @property {() => Promise} [resolvePlatformArch] + * Override platform-arch resolution. Default calls getCurrentPlatformArch() + * from platform-mappings (returns e.g. 'darwin-arm64'). Platform-agnostic + * builds (e.g. JS bundling in socket-tui) should return a fixed string + * like 'universal' so the cache key stays stable across host OSes. + */ + +async function readJson(filePath) { + let raw + try { + raw = await fs.readFile(filePath, 'utf8') + } catch (e) { + if (e.code === 'ENOENT') { + return undefined + } + throw new Error(`Failed to read ${filePath}: ${errorMessage(e)}`, { + cause: e, + }) + } + try { + return JSON.parse(raw) + } catch (e) { + throw new Error(`Failed to parse ${filePath}: ${errorMessage(e)}`, { + cause: e, + }) + } +} + +async function loadExternalTools(packageRoot) { + const filePath = path.join(packageRoot, 'external-tools.json') + const data = await readJson(filePath) + if (!data) { + return { versions: {}, rawHash: '' } + } + validateExternalTools(data) + const versions = {} + for (const [tool, meta] of Object.entries(data.tools ?? {})) { + versions[tool] = meta?.version ?? '' + } + const rawHash = createHash('sha256') + .update(JSON.stringify(data)) + .digest('hex') + .slice(0, 16) + return { versions, rawHash } +} + +async function loadPackageJson(packageRoot) { + const pkg = await readJson(path.join(packageRoot, 'package.json')) + if (!pkg) { + throw new Error(`Missing package.json in ${packageRoot}`) + } + return pkg +} + +function hashFileContents(files) { + const hash = createHash('sha256') + for (const file of files.toSorted()) { + let content = Buffer.alloc(0) + if (existsSync(file)) { + try { + content = readFileSync(file) + } catch {} + } + hash.update(`${file}:`) + hash.update(content) + } + return hash.digest('hex').slice(0, 16) +} + +function buildCacheKey({ + buildMode, + nodeVersion, + platformArch, + sources, + toolVersions, + toolsHash, + packageVersion, + extraHash, +}) { + const hash = createHash('sha256') + hash.update(`node=${nodeVersion}`) + hash.update(`platformArch=${platformArch}`) + hash.update(`mode=${buildMode}`) + hash.update(`tools=${toolsHash}`) + for (const tool of Object.keys(toolVersions).toSorted()) { + hash.update(`${tool}@${toolVersions[tool]}`) + } + for (const key of Object.keys(sources).toSorted()) { + const src = sources[key] ?? {} + hash.update( + `src:${key}=${src.version ?? ''}:${src.ref ?? ''}:${src.url ?? ''}`, + ) + } + if (extraHash) { + hash.update(`extra=${extraHash}`) + } + const digest = hash.digest('hex').slice(0, 12) + return `v${nodeVersion}-${platformArch}-${buildMode}-${digest}-${packageVersion}` +} + +function parseFlags(argv) { + const args = new Set(argv) + const getValue = flag => { + const prefix = `${flag}=` + for (const arg of argv) { + if (arg.startsWith(prefix)) { + return arg.slice(prefix.length) + } + } + return undefined + } + return { + force: args.has('--force'), + clean: args.has('--clean'), + printCacheKey: args.has('--cache-key'), + cleanStage: getValue('--clean-stage'), + fromStage: getValue('--from-stage'), + raw: args, + } +} + +function resolveCheckpointBuildDir(stage, ctx) { + if (stage.shared && ctx.sharedPaths?.buildDir) { + return ctx.sharedPaths.buildDir + } + return ctx.paths.buildDir +} + +async function runStage(stage, ctx, stageParams) { + const { buildMode, forceRebuild, logger } = ctx + + if (stage.skipInDev && buildMode === 'dev') { + logger.substep(`Skipping ${stage.name} (dev build)`) + return + } + + if (typeof stage.skip === 'function' && stage.skip(ctx)) { + logger.substep(`Skipping ${stage.name} (skip predicate)`) + return + } + + const buildDir = resolveCheckpointBuildDir(stage, ctx) + const sourcePaths = [ + path.join(ctx.packageRoot, 'external-tools.json'), + path.join(ctx.packageRoot, 'package.json'), + ...(stage.sourcePaths ?? []), + ].filter(p => existsSync(p)) + + const platformMeta = stage.shared + ? {} + : { + buildMode, + nodeVersion: ctx.nodeVersion, + platform: process.platform, + arch: process.arch, + } + + const shouldProceed = await shouldRun( + buildDir, + '', + stage.name, + forceRebuild, + sourcePaths, + platformMeta, + ) + + if (!shouldProceed) { + logger.substep(`✓ ${stage.name} up-to-date (cached)`) + return + } + + logger.step(`Running ${stage.name}`) + const result = (await stage.run(ctx, stageParams)) ?? {} + const { + artifactPath, + binaryPath, + binarySize, + smokeTest = async () => {}, + } = result + + await createCheckpoint(buildDir, stage.name, smokeTest, { + ...(artifactPath ? { artifactPath } : {}), + ...(binaryPath ? { binaryPath } : {}), + ...(binarySize !== undefined ? { binarySize } : {}), + packageRoot: ctx.packageRoot, + sourcePaths, + ...platformMeta, + }) +} + +/** + * Validate + run a pipeline. On --cache-key, prints the key and exits without + * building. Returns the context so the caller can render a summary. + * + * @param {RunPipelineOptions} options + * @param {object} [cliOverrides] - Pre-parsed flags (for programmatic use). + * @returns {Promise} + */ +export async function runPipeline(options, cliOverrides) { + const { + extraCacheInputs = [], + getBuildPaths, + getOutputFiles, + getSharedBuildPaths, + packageName, + packageRoot, + preflight, + resolvePlatformArch, + stages, + } = options + + const flags = cliOverrides ?? parseFlags(process.argv.slice(2)) + const buildMode = getBuildMode(flags.raw ?? new Set()) + const platformArch = resolvePlatformArch + ? await resolvePlatformArch() + : await getCurrentPlatformArch() + const nodeVersion = getNodeVersion().replace(/^v/, '') + + const [pkgJson, { versions: toolVersions, rawHash: toolsHash }] = + await Promise.all([ + loadPackageJson(packageRoot), + loadExternalTools(packageRoot), + ]) + + const sources = pkgJson.sources ?? {} + const packageVersion = pkgJson.version ?? '0.0.0' + + const extraHash = + extraCacheInputs.length > 0 ? hashFileContents(extraCacheInputs) : '' + const cacheKey = buildCacheKey({ + buildMode, + extraHash, + nodeVersion, + packageVersion, + platformArch, + sources, + toolsHash, + toolVersions, + }) + + if (flags.printCacheKey) { + process.stdout.write(`${cacheKey}\n`) + return /** @type {any} */ (undefined) + } + + const paths = getBuildPaths(buildMode, platformArch) + const sharedPaths = getSharedBuildPaths ? getSharedBuildPaths() : undefined + const outputFiles = getOutputFiles ? getOutputFiles(paths) : [] + + // Validate chain for typos / unknown names. + validateCheckpointChain( + stages.map(s => s.name), + packageName, + ) + + const ctx = { + buildMode, + cacheKey, + forceRebuild: flags.force, + logger, + nodeVersion, + packageName, + packageRoot, + paths, + platformArch, + sharedPaths, + sources, + toolVersions, + } + + const totalStart = Date.now() + logger.step(`🔨 Building ${packageName}`) + logger.info(`Mode: ${buildMode}`) + logger.info(`Platform: ${platformArch}`) + logger.info(`Cache key: ${cacheKey}`) + logger.info('') + + // Handle --clean / --clean-stage / missing-output clean-up. + if (flags.clean) { + logger.substep('Clean build requested — removing all checkpoints') + await cleanCheckpoint(paths.buildDir, '') + if (sharedPaths?.buildDir) { + await cleanCheckpoint(sharedPaths.buildDir, '') + } + } else if (flags.cleanStage) { + logger.substep(`Clean requested for stage: ${flags.cleanStage}`) + // Invalidates this stage + anything depending on it. + const idx = stages.findIndex(s => s.name === flags.cleanStage) + if (idx === -1) { + throw new Error( + `Unknown --clean-stage=${flags.cleanStage}. Valid: ${stages.map(s => s.name).join(', ')}`, + ) + } + for (const stage of stages.slice(idx)) { + const buildDir = resolveCheckpointBuildDir(stage, ctx) + const markerDir = path.join(buildDir, 'checkpoints') + for (const ext of ['.json', '.tar.gz', '.tar.gz.lock']) { + const file = path.join(markerDir, `${stage.name}${ext}`) + if (existsSync(file)) { + await fs.rm(file, { force: true }) + } + } + } + } else if (outputFiles.length && outputFiles.some(p => !existsSync(p))) { + logger.substep( + 'Output artifacts missing — invalidating all checkpoints to rebuild', + ) + await cleanCheckpoint(paths.buildDir, '') + if (sharedPaths?.buildDir) { + await cleanCheckpoint(sharedPaths.buildDir, '') + } + } + + if (preflight) { + logger.step('Pre-flight Checks') + await preflight() + logger.success('Pre-flight checks passed') + } + + // --from-stage: pretend earlier stages succeeded (they should have cached + // checkpoints already). We just skip running them. + let startIdx = 0 + if (flags.fromStage) { + startIdx = stages.findIndex(s => s.name === flags.fromStage) + if (startIdx === -1) { + throw new Error( + `Unknown --from-stage=${flags.fromStage}. Valid: ${stages.map(s => s.name).join(', ')}`, + ) + } + logger.substep(`Starting from stage: ${flags.fromStage}`) + } + + for (const stage of stages.slice(startIdx)) { + await runStage(stage, ctx, {}) + } + + const seconds = ((Date.now() - totalStart) / 1000).toFixed(1) + logger.step('🎉 Build Complete!') + logger.success(`Total time: ${seconds}s`) + logger.success(`Output: ${paths.outputFinalDir ?? paths.buildDir}`) + if (outputFiles.length) { + logger.info('') + logger.info('Files:') + for (const file of outputFiles) { + logger.info(` - ${path.relative(packageRoot, file)}`) + } + logger.info('') + } + return ctx +} + +/** + * CLI entry-point helper. Wraps runPipeline with a top-level error handler. + * @param {RunPipelineOptions} options + */ +export async function runPipelineCli(options) { + try { + await runPipeline(options) + } catch (e) { + logger.error(errorMessage(e)) + process.exitCode = 1 + throw e + } +} diff --git a/packages/build-infra/lib/checkpoint-manager.mts b/packages/build-infra/lib/checkpoint-manager.mts new file mode 100644 index 000000000..48a598bc1 --- /dev/null +++ b/packages/build-infra/lib/checkpoint-manager.mts @@ -0,0 +1,252 @@ +/** + * Build checkpoint manager (lean). + * + * Same public API as socket-btm's checkpoint-manager but sized for the + * single-stage wasm builds in this repo (lang/{rust,cpp,go}). Each stage + * writes a JSON marker `{ name }.json` keyed by a content hash of its + * source inputs + platform/arch/mode. If the hash matches next run, the + * stage is skipped. + * + * What this intentionally omits vs socket-btm: + * - Tarball archival (socket-btm archives the built artifact so CI can + * restore it between jobs; lang wasm rebuilds take seconds, not 30 min). + * - Ad-hoc macOS codesign (wasm artifacts don't need it). + * - Cross-process atomic-write ceremony (no concurrent CI jobs racing on + * the same build dir in this repo). + * - restoreCheckpoint (nothing to restore when there's no tarball). + * + * Exports mirror the names build-pipeline consumes, so the orchestrator is + * identical across repos. + */ + +import { createHash } from 'node:crypto' +import { existsSync, promises as fs, readFileSync } from 'node:fs' +import path from 'node:path' +import process from 'node:process' + +import { errorMessage } from '@socketsecurity/lib/errors' +import { safeDelete, safeMkdir } from '@socketsecurity/lib/fs' +import { getDefaultLogger } from '@socketsecurity/lib/logger' + +const logger = getDefaultLogger() + +function checkpointDir(buildDir, packageName) { + return packageName + ? path.join(buildDir, 'checkpoints', packageName) + : path.join(buildDir, 'checkpoints') +} + +function checkpointFile(buildDir, packageName, name) { + return path.join(checkpointDir(buildDir, packageName), `${name}.json`) +} + +function hashSourcePaths(sourcePaths) { + const hash = createHash('sha256') + for (const file of [...sourcePaths].sort()) { + hash.update(`${file}:`) + if (existsSync(file)) { + try { + hash.update(readFileSync(file)) + } catch (e) { + if (e.code !== 'ENOENT') { + throw e + } + } + } + } + return hash.digest('hex') +} + +function platformCacheKey({ buildMode, nodeVersion, platform, arch, libc }) { + const parts = [ + buildMode && `mode=${buildMode}`, + nodeVersion && `node=${nodeVersion}`, + platform && `platform=${platform}`, + arch && `arch=${arch}`, + libc && `libc=${libc}`, + ].filter(Boolean) + if (!parts.length) { + return '' + } + return createHash('sha256').update(parts.join('|')).digest('hex').slice(0, 16) +} + +function computeCacheHash(sourcePaths, options) { + const sourcesHash = sourcePaths?.length ? hashSourcePaths(sourcePaths) : '' + const platformHash = platformCacheKey(options || {}) + if (!sourcesHash && !platformHash) { + return '' + } + return createHash('sha256') + .update(sourcesHash) + .update('|') + .update(platformHash) + .digest('hex') +} + +/** + * Does a checkpoint JSON marker exist? + */ +export function hasCheckpoint(buildDir, packageName, name) { + return existsSync(checkpointFile(buildDir, packageName, name)) +} + +/** + * Read a checkpoint's JSON data, or undefined if it does not exist. + */ +export async function getCheckpointData(buildDir, packageName, name) { + const file = checkpointFile(buildDir, packageName, name) + if (!existsSync(file)) { + return undefined + } + try { + return JSON.parse(await fs.readFile(file, 'utf8')) + } catch (e) { + logger.warn( + `Checkpoint ${name} JSON unreadable (${errorMessage(e)}) — ignoring`, + ) + return undefined + } +} + +/** + * Run `smokeTest`, then write a checkpoint JSON marker. + * + * @param {string} buildDir + * @param {string} name - Checkpoint name (must be a CHECKPOINTS value). + * @param {() => Promise} smokeTest - Throws if the stage output is invalid. + * @param {object} [options] + * @param {string} [options.packageName] + * @param {string} [options.artifactPath] - Informational; recorded in JSON. + * @param {string} [options.binaryPath] - Informational; recorded in JSON. + * @param {string|number} [options.binarySize] - Informational; recorded in JSON. + * @param {string[]} [options.sourcePaths] - Inputs hashed into the cache key. + * @param {string} [options.buildMode] + * @param {string} [options.nodeVersion] + * @param {string} [options.platform] + * @param {string} [options.arch] + * @param {string} [options.libc] + * @param {string} [options.packageRoot] + */ +export async function createCheckpoint( + buildDir, + name, + smokeTest, + options = {}, +) { + if (typeof smokeTest !== 'function') { + throw new Error( + `createCheckpoint('${name}'): expected smokeTest callback as argument 3, got ${typeof smokeTest}.`, + ) + } + + const { + arch, + artifactPath, + binaryPath, + binarySize, + buildMode, + libc, + nodeVersion, + packageName = '', + packageRoot, + platform, + sourcePaths, + } = options + + try { + await smokeTest() + } catch (e) { + throw new Error( + `Smoke test failed for checkpoint '${name}': ${errorMessage(e)}`, + { cause: e }, + ) + } + + const dir = checkpointDir(buildDir, packageName) + await safeMkdir(dir) + + const cacheHash = computeCacheHash(sourcePaths, { + arch, + buildMode, + libc, + nodeVersion, + platform, + }) + + const data = { + name, + createdAt: new Date().toISOString(), + cacheHash, + artifactPath, + binaryPath, + binarySize, + platform, + arch, + libc, + buildMode, + nodeVersion, + } + + const file = checkpointFile(buildDir, packageName, name) + await fs.writeFile(file, `${JSON.stringify(data, null, 2)}\n`, 'utf8') + + const relRoot = packageRoot ? path.relative(packageRoot, file) : file + logger.substep(`✓ Checkpoint ${name} written (${relRoot})`) +} + +/** + * Should the stage run? True if force, no checkpoint, missing cache hash, + * or the hash no longer matches current inputs. + */ +export async function shouldRun( + buildDir, + packageName, + name, + force = false, + sourcePaths, + options = {}, +) { + if (force) { + return true + } + if (!hasCheckpoint(buildDir, packageName, name)) { + return true + } + + // Only validate hash if the caller provided inputs or platform metadata. + const wantsValidation = + (sourcePaths && sourcePaths.length) || + options.buildMode || + options.platform || + options.arch + + if (!wantsValidation) { + return false + } + + const data = await getCheckpointData(buildDir, packageName, name) + if (!data) { + return true + } + + const expected = computeCacheHash(sourcePaths, options) + if (!data.cacheHash || data.cacheHash !== expected) { + logger.substep(`Checkpoint ${name} stale (cache hash changed) — rebuilding`) + return true + } + + return false +} + +/** + * Delete all checkpoints under a build dir (or a single package's scope). + */ +export async function cleanCheckpoint(buildDir, packageName) { + const dir = checkpointDir(buildDir, packageName) + if (!existsSync(dir)) { + return + } + await safeDelete(dir) + logger.substep('Checkpoints cleaned') +} diff --git a/packages/build-infra/lib/constants.mts b/packages/build-infra/lib/constants.mts new file mode 100644 index 000000000..33bc9ec70 --- /dev/null +++ b/packages/build-infra/lib/constants.mts @@ -0,0 +1,109 @@ +/** + * Shared constants for the build-pipeline orchestrator (socket-cli variant). + * + * Mirrors the socket-btm/ultrathink/socket-tui/sdxgen API surface + * (BUILD_STAGES, CHECKPOINTS, CHECKPOINT_CHAINS, validateCheckpointChain, + * getBuildMode). socket-cli doesn't build wasm — it consumes pre-built + * wasm + node binaries from socket-btm — so the orchestrator name + * ('build-pipeline') is historical; the machinery is build-type-agnostic. + */ + +import process from 'node:process' + +import { getCI } from '@socketsecurity/lib/env/ci' + +/** + * Build stage directory names inside build//. + */ +export const BUILD_STAGES = { + BUNDLED: 'Bundled', + FINAL: 'Final', + OPTIMIZED: 'Optimized', + RELEASE: 'Release', + STRIPPED: 'Stripped', + SEA: 'Sea', + SYNC: 'Sync', + TYPES: 'Types', +} + +/** + * Canonical checkpoint names. Each pipeline stage picks one. + */ +export const CHECKPOINTS = { + CLI: 'cli', + FINALIZED: 'finalized', + SEA: 'sea', +} + +const VALID_CHECKPOINT_VALUES = new Set(Object.values(CHECKPOINTS)) + +/** + * Checkpoint chain for socket-cli's build pipeline. + * Order: newest → oldest (matching socket-btm convention). + * + * The SEA binary is built only for --force / --prod today; the chain is + * declared including SEA so --clean-stage=sea works when it runs. + */ +export const CHECKPOINT_CHAINS = { + cli: () => [CHECKPOINTS.FINALIZED, CHECKPOINTS.SEA, CHECKPOINTS.CLI], +} + +/** + * Validate a checkpoint chain at runtime. + */ +export function validateCheckpointChain( + chain: string[], + packageName: string, +) { + if (!Array.isArray(chain)) { + throw new Error(`${packageName}: Checkpoint chain must be an array`) + } + if (chain.length === 0) { + throw new Error(`${packageName}: Checkpoint chain cannot be empty`) + } + const invalid = chain.filter(cp => !VALID_CHECKPOINT_VALUES.has(cp)) + if (invalid.length) { + throw new Error( + `${packageName}: Invalid checkpoint names in chain: ${invalid.join(', ')}. ` + + `Valid: ${Object.keys(CHECKPOINTS).join(', ')}`, + ) + } + const seen = new Set() + for (const cp of chain) { + if (seen.has(cp)) { + throw new Error(`${packageName}: Duplicate checkpoint in chain: ${cp}`) + } + seen.add(cp) + } +} + +// Validate chain registry at module load. +for (const [name, generator] of Object.entries(CHECKPOINT_CHAINS)) { + validateCheckpointChain(generator(), `CHECKPOINT_CHAINS.${name}`) +} + +/** + * Resolve the build mode from CLI flags, env, or CI autodetect. + */ +export function getBuildMode(args?: string[] | Set): string { + if (args) { + const has = Array.isArray(args) + ? (flag: string) => args.includes(flag) + : (flag: string) => args.has(flag) + if (has('--prod')) { + return 'prod' + } + if (has('--dev')) { + return 'dev' + } + } + if (process.env['BUILD_MODE']) { + return process.env['BUILD_MODE'] + } + return getCI() ? 'prod' : 'dev' +} + +/** + * Path used by platform-mappings.isMusl() for Alpine detection. + */ +export const ALPINE_RELEASE_FILE = '/etc/alpine-release' diff --git a/packages/build-infra/lib/external-tools-schema.json b/packages/build-infra/lib/external-tools-schema.json new file mode 100644 index 000000000..ee9d051e8 --- /dev/null +++ b/packages/build-infra/lib/external-tools-schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Schema for external-tools.json files", + "type": "object", + "properties": { + "$schema": { "type": "string" }, + "description": { "type": "string" }, + "extends": { + "type": "string", + "description": "Path to a base external-tools.json to inherit from" + }, + "tools": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "description": { "type": "string" }, + "version": { "type": "string" }, + "packageManager": { + "type": "string", + "enum": ["npm", "pip", "pnpm"] + }, + "notes": { + "oneOf": [ + { "type": "string" }, + { "type": "array", "items": { "type": "string" } } + ] + }, + "repository": { "type": "string" }, + "release": { "type": "string", "enum": ["asset", "archive"] }, + "tag": { "type": "string" }, + "checksums": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "object", + "properties": { + "asset": { "type": "string" }, + "sha256": { "type": "string" } + }, + "required": ["asset", "sha256"] + }, + { "type": "string" } + ] + } + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/packages/build-infra/lib/external-tools-schema.mts b/packages/build-infra/lib/external-tools-schema.mts new file mode 100644 index 000000000..00031aef4 --- /dev/null +++ b/packages/build-infra/lib/external-tools-schema.mts @@ -0,0 +1,124 @@ +/** + * TypeBox schema for external-tools.json files. + * + * Validates tool configuration used by the tool-installer to auto-download + * and verify external build dependencies. + * + * Normalized schema across all Socket repos: + * socket-btm: build tools (system tools, pip packages) + * socket-cli: bundle tools (npm packages, GitHub release binaries) + * socket-registry: CI tools (GitHub release binaries) + * ultrathink: build tools (compilers, language toolchains) + */ + +import { Type } from '@sinclair/typebox' + +import { validateSchema } from '@socketsecurity/lib/schema/validate' + +const toolSchema = Type.Object( + { + // Common fields (all repos). + description: Type.Optional( + Type.String({ description: 'What the tool is used for' }), + ), + version: Type.Optional( + Type.String({ + description: 'Version requirement (exact "0.15.2" or range "3.28+")', + }), + ), + packageManager: Type.Optional( + Type.Union( + [Type.Literal('npm'), Type.Literal('pip'), Type.Literal('pnpm')], + { + description: 'Package manager for installation. Absent = system tool', + }, + ), + ), + notes: Type.Optional( + Type.Union([Type.String(), Type.Array(Type.String())], { + description: 'Additional notes about the tool', + }), + ), + + // GitHub release fields (socket-cli bundle-tools, socket-registry). + repository: Type.Optional( + Type.String({ description: 'Repository in "github:owner/repo" format' }), + ), + release: Type.Optional( + Type.Union([Type.Literal('asset'), Type.Literal('archive')], { + description: + 'Release type: "asset" for individual binaries, "archive" for source tarballs', + }), + ), + tag: Type.Optional( + Type.String({ description: 'Release tag (when different from version)' }), + ), + checksums: Type.Optional( + Type.Record( + Type.String(), + Type.Union([ + // Platform-keyed: { "darwin-arm64": { "asset": "...", "sha256": "..." } } + Type.Object({ + asset: Type.String(), + sha256: Type.String(), + }), + // Flat: { "file.tar.gz": "abc..." } (legacy/simple). + Type.String(), + ]), + { description: 'Checksums keyed by platform or asset filename' }, + ), + ), + + // npm package fields (socket-cli bundle-tools). + integrity: Type.Optional( + Type.String({ description: 'npm package integrity hash (sha512)' }), + ), + npm: Type.Optional( + Type.Object( + { + package: Type.Optional(Type.String()), + version: Type.Optional(Type.String()), + }, + { + description: + 'Nested npm package reference (when tool has both binary and npm forms)', + }, + ), + ), + }, + // TypeBox equivalent of Zod's .passthrough() — allow extra properties. + { additionalProperties: true }, +) + +export const externalToolsSchema = Type.Object( + { + $schema: Type.Optional(Type.String()), + description: Type.Optional( + Type.String({ + description: 'Human-readable description of this config file', + }), + ), + extends: Type.Optional( + Type.String({ + description: 'Path to a base external-tools.json to inherit tools from', + }), + ), + tools: Type.Optional( + Type.Record(Type.String(), toolSchema, { + description: 'Map of tool name to tool configuration', + }), + ), + }, + { additionalProperties: true }, +) + +/** + * Validate an external-tools.json object against the schema. + * + * @param {unknown} data - Parsed JSON data. + * @returns `{ ok: true, value }` on success, `{ ok: false, errors }` with + * normalized `{ path, message }` issues on failure. + */ +export function validateExternalTools(data) { + return validateSchema(externalToolsSchema, data) +} diff --git a/packages/build-infra/lib/platform-mappings.mts b/packages/build-infra/lib/platform-mappings.mts new file mode 100644 index 000000000..245fe6376 --- /dev/null +++ b/packages/build-infra/lib/platform-mappings.mts @@ -0,0 +1,234 @@ +import process from 'node:process' + +/** + * Shared platform and architecture mappings for GitHub release assets. + * + * Maps Node.js platform/architecture names to release asset naming conventions. + * Used consistently across all download and build scripts to avoid duplication. + */ + +import { existsSync } from 'node:fs' + +import { getDefaultLogger } from '@socketsecurity/lib/logger' +import { spawn } from '@socketsecurity/lib/spawn' + +import { ALPINE_RELEASE_FILE } from './constants.mts' + +const logger = getDefaultLogger() + +/** + * Maps Node.js platform names to GitHub release platform names. + * + * @type {Readonly>} + */ +export const RELEASE_PLATFORM_MAP = Object.freeze({ + __proto__: null, + darwin: 'darwin', + linux: 'linux', + win32: 'win', +}) + +/** + * Maps Node.js architecture names to GitHub release architecture names. + * + * @type {Readonly>} + */ +export const RELEASE_ARCH_MAP = Object.freeze({ + __proto__: null, + arm64: 'arm64', + ia32: 'x86', + x64: 'x64', +}) + +/** + * Get platform-arch string for internal directory paths (download locations). + * Uses Node.js platform naming directly (win32, darwin, linux). + * + * @param {string} platform - Node.js platform (darwin, linux, win32). + * @param {string} arch - Node.js architecture (arm64, x64, ia32). + * @param {string|undefined} [libc] - C library variant (musl, glibc) - Linux only. + * @returns {string} Platform-arch string (e.g., 'win32-x64', 'linux-x64-musl'). + * @throws {Error} If platform/arch is unsupported. + */ +export function getPlatformArch(platform, arch, libc) { + const releaseArch = RELEASE_ARCH_MAP[arch] + + if (!releaseArch) { + throw new Error(`Unsupported arch: ${arch}`) + } + if (platform !== 'darwin' && platform !== 'linux' && platform !== 'win32') { + throw new Error(`Unsupported platform: ${platform}`) + } + + // Validate libc parameter. + if (libc && libc !== 'musl' && libc !== 'glibc') { + throw new Error(`Invalid libc: ${libc}. Valid options: musl, glibc`) + } + if (libc && platform !== 'linux') { + throw new Error( + `libc parameter is only valid for Linux platform (got platform: ${platform})`, + ) + } + + // Add musl suffix for Linux musl builds. + const muslSuffix = platform === 'linux' && libc === 'musl' ? '-musl' : '' + // Use Node.js platform naming directly for directory paths + return `${platform}-${releaseArch}${muslSuffix}` +} + +/** + * Get platform-arch string for GitHub release asset naming. + * Uses shortened platform names (win instead of win32). + * + * @param {string} platform - Node.js platform (darwin, linux, win32). + * @param {string} arch - Node.js architecture (arm64, x64, ia32). + * @param {string|undefined} [libc] - C library variant (musl, glibc) - Linux only. + * @returns {string} Platform-arch string for assets (e.g., 'win-x64', 'linux-x64-musl'). + * @throws {Error} If platform/arch is unsupported. + */ +export function getAssetPlatformArch(platform, arch, libc) { + const releasePlatform = RELEASE_PLATFORM_MAP[platform] + const releaseArch = RELEASE_ARCH_MAP[arch] + + if (!releasePlatform || !releaseArch) { + throw new Error(`Unsupported platform/arch: ${platform}/${arch}`) + } + + // Validate libc parameter. + if (libc && libc !== 'musl' && libc !== 'glibc') { + throw new Error(`Invalid libc: ${libc}. Valid options: musl, glibc`) + } + if (libc && platform !== 'linux') { + throw new Error( + `libc parameter is only valid for Linux platform (got platform: ${platform})`, + ) + } + // Warn when libc is missing for Linux - this usually indicates a bug. + // Use getCurrentPlatformArch() instead, which auto-detects libc. + if (platform === 'linux' && libc === undefined) { + logger.warn( + 'getAssetPlatformArch() called for Linux without libc parameter. ' + + 'This may cause builds to output to wrong directory (linux-x64 vs linux-x64-musl). ' + + 'Consider using getCurrentPlatformArch() which auto-detects libc.', + ) + } + + // Add musl suffix for Linux musl builds. + const muslSuffix = platform === 'linux' && libc === 'musl' ? '-musl' : '' + // Use shortened platform names for asset names + return `${releasePlatform}-${releaseArch}${muslSuffix}` +} + +/** + * Detect if running on musl libc (Alpine Linux). + * + * @returns {Promise} True if running on musl libc. + */ +export async function isMusl() { + if (process.platform !== 'linux') { + return false + } + + // Check for Alpine release file. + if (existsSync(ALPINE_RELEASE_FILE)) { + return true + } + + // Check ldd version for musl. + try { + const result = await spawn('ldd', ['--version'], { stdio: 'pipe' }) + const output = result.stdout + result.stderr + return output.includes('musl') + } catch { + // Expected: ldd may not exist in some environments. + return false + } +} + +/** + * Get platform-arch string for the current platform using shared mapping. + * + * Resolution order: + * 1. `PLATFORM_ARCH` env — the explicit value the workflow/Dockerfile injected + * (set by .github/workflows/*.yml build-args and every Dockerfile). + * 2. Cross-compile env (`TARGET_ARCH`, `LIBC`) applied on top of the host's + * platform/arch. + * 3. Full auto-detect via `isMusl()` + `process.arch` + `process.platform`. + * + * @returns {Promise} Platform-arch string (e.g., 'win-x64', 'linux-x64-musl'). + */ +export async function getCurrentPlatformArch() { + // If the workflow or Dockerfile set PLATFORM_ARCH explicitly, trust it. + if (process.env.PLATFORM_ARCH) { + return process.env.PLATFORM_ARCH + } + // Respect LIBC environment variable for cross-compilation (set by workflows) + // Falls back to isMusl() for host detection when not cross-compiling. + const libc = process.env.LIBC || ((await isMusl()) ? 'musl' : undefined) + // Respect TARGET_ARCH for cross-compilation (set by workflows/Makefiles) + const arch = process.env.TARGET_ARCH || process.arch + return getAssetPlatformArch(process.platform, arch, libc) +} + +/** + * Read the requested glibc floor from the GLIBC_FLOOR env var. + * + * Returned value is a string like "2.17" or "2.28", or undefined when unset. + * No behavior change today — this is groundwork for threading a glibc floor + * dimension through cache keys and Docker image selection when we lower the + * floor. See packages/node-smol-builder/docs/plans/glibc-floor-lowering.md. + * + * Callers should treat `undefined` as "fall back to the repo's current default + * build image (glibc 2.28)" so behavior is unchanged until the env is set. + * + * @returns {string | undefined} Requested glibc floor, or undefined. + */ +export function getRequestedGlibcFloor(): string | undefined { + const raw = process.env.GLIBC_FLOOR + if (!raw) { + return undefined + } + const trimmed = raw.trim() + // Accept "2.17" or "2.28". Reject anything else so typos surface loudly. + if (trimmed === '2.17' || trimmed === '2.28') { + return trimmed + } + throw new Error( + `Unrecognized GLIBC_FLOOR="${raw}". Expected "2.17" or "2.28".`, + ) +} + +/** + * Check if tar supports --no-absolute-names (GNU tar has it, busybox tar doesn't). + * + * @returns {Promise} True if tar supports --no-absolute-names. + */ +export async function tarSupportsNoAbsoluteNames() { + try { + const result = await spawn('tar', ['--help'], { stdio: 'pipe' }) + return (result.stdout || '').includes('--no-absolute-names') + } catch { + return false + } +} + +/** + * Check if tar supports --overwrite (GNU tar has it, BSD/macOS tar doesn't). + * + * @returns {Promise} True if tar supports --overwrite. + */ +export async function tarSupportsOverwrite() { + // BSD tar on macOS doesn't support --overwrite. + // Quick platform check to avoid spawning tar unnecessarily. + if (process.platform === 'darwin') { + return false + } + try { + const result = await spawn('tar', ['--help'], { stdio: 'pipe' }) + // Look for the actual --overwrite flag, not just the word "overwrite" + // (BSD tar mentions "overwrite" in the -k flag description but doesn't support --overwrite) + return /^\s*--overwrite\b/m.test(result.stdout || '') + } catch { + return false + } +} diff --git a/packages/build-infra/lib/version-helpers.mts b/packages/build-infra/lib/version-helpers.mts new file mode 100644 index 000000000..4ce2f3d49 --- /dev/null +++ b/packages/build-infra/lib/version-helpers.mts @@ -0,0 +1,56 @@ +/** + * Version helpers used by the build-pipeline orchestrator. + * + * The socket-btm version of this file also fetches Node.js release + * checksums and extracts submodule SHAs for its native-binary builders; + * ultrathink's lang wasm pipelines don't need any of that. Keep only + * getNodeVersion and getToolVersion (the two the orchestrator imports). + */ + +import { promises as fs } from 'node:fs' +import path from 'node:path' +import process from 'node:process' + +import { errorMessage } from '@socketsecurity/lib/errors' + +/** + * The Node.js version running this process, without the leading "v". + */ +export function getNodeVersion(): string { + return process.version.replace(/^v/, '') +} + +/** + * Read a pinned tool version from the package's external-tools.json. + * + * @throws when the file is missing or the tool has no version recorded. + */ +export async function getToolVersion( + packageRoot: string, + toolName: string, +): Promise { + const filePath = path.join(packageRoot, 'external-tools.json') + let raw: string + try { + raw = await fs.readFile(filePath, 'utf8') + } catch (e) { + throw new Error(`Failed to read ${filePath}: ${errorMessage(e)}`, { + cause: e, + }) + } + let data: any + try { + data = JSON.parse(raw) + } catch (e) { + throw new Error(`Failed to parse ${filePath}: ${errorMessage(e)}`, { + cause: e, + }) + } + const version = data?.tools?.[toolName]?.version + if (!version) { + throw new Error( + `external-tools.json in ${packageRoot} has no version pinned for "${toolName}".`, + ) + } + return version +} diff --git a/packages/build-infra/package.json b/packages/build-infra/package.json index 9a962386f..c7af6f767 100644 --- a/packages/build-infra/package.json +++ b/packages/build-infra/package.json @@ -5,17 +5,24 @@ "private": true, "type": "module", "exports": { + "./lib/checkpoint-manager": "./lib/checkpoint-manager.mts", + "./lib/constants": "./lib/constants.mts", "./lib/esbuild-helpers": "./lib/esbuild-helpers.mts", "./lib/esbuild-plugin-unicode-transform": "./lib/esbuild-plugin-unicode-transform.mts", + "./lib/external-tools-schema": "./lib/external-tools-schema.mts", "./lib/extraction-cache": "./lib/extraction-cache.mts", "./lib/github-error-utils": "./lib/github-error-utils.mts", "./lib/github-releases": "./lib/github-releases.mts", + "./lib/platform-mappings": "./lib/platform-mappings.mts", "./lib/platform-targets": "./lib/platform-targets.mts", - "./lib/unicode-property-escape-transform": "./lib/unicode-property-escape-transform.mts" + "./lib/unicode-property-escape-transform": "./lib/unicode-property-escape-transform.mts", + "./lib/version-helpers": "./lib/version-helpers.mts", + "./lib/build-pipeline": "./lib/build-pipeline.mts" }, "dependencies": { "@babel/parser": "catalog:", "@babel/traverse": "catalog:", + "@sinclair/typebox": "catalog:", "@socketsecurity/lib": "catalog:", "magic-string": "catalog:" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index adc83df24..79a9d9cd6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,6 +72,9 @@ catalogs: '@pnpm/logger': specifier: 1001.0.0 version: 1001.0.0 + '@sinclair/typebox': + specifier: 0.34.49 + version: 0.34.49 '@socketregistry/hyrious__bun.lockb': specifier: 1.0.19 version: 1.0.19 @@ -254,7 +257,7 @@ overrides: '@octokit/graphql': 9.0.1 '@octokit/request-error': 7.0.0 '@sigstore/sign': 4.1.0 - '@socketsecurity/lib': 5.21.0 + '@socketsecurity/lib': 5.24.0 aggregate-error: npm:@socketregistry/aggregate-error@^1.0.15 ansi-regex: 6.2.2 brace-expansion: 5.0.5 @@ -393,8 +396,8 @@ importers: specifier: 'catalog:' version: 3.0.1 '@socketsecurity/lib': - specifier: 5.21.0 - version: 5.21.0(typescript@5.9.3) + specifier: 5.24.0 + version: 5.24.0(typescript@5.9.3) '@socketsecurity/registry': specifier: 'catalog:' version: 2.0.2(typescript@5.9.3) @@ -573,8 +576,8 @@ importers: specifier: 'catalog:' version: 1.4.2 '@socketsecurity/lib': - specifier: 5.21.0 - version: 5.21.0(typescript@5.9.3) + specifier: 5.24.0 + version: 5.24.0(typescript@5.9.3) '@socketsecurity/sdk': specifier: 'catalog:' version: 4.0.1 @@ -586,8 +589,8 @@ importers: .claude/hooks/setup-security-tools: dependencies: '@socketsecurity/lib': - specifier: 5.21.0 - version: 5.21.0(typescript@5.9.3) + specifier: 5.24.0 + version: 5.24.0(typescript@5.9.3) packages/build-infra: dependencies: @@ -597,9 +600,12 @@ importers: '@babel/traverse': specifier: 'catalog:' version: 7.28.4 + '@sinclair/typebox': + specifier: 'catalog:' + version: 0.34.49 '@socketsecurity/lib': - specifier: 5.21.0 - version: 5.21.0(typescript@5.9.3) + specifier: 5.24.0 + version: 5.24.0(typescript@5.9.3) magic-string: specifier: 'catalog:' version: 0.30.19 @@ -655,8 +661,8 @@ importers: specifier: 'catalog:' version: 3.0.1 '@socketsecurity/lib': - specifier: 5.21.0 - version: 5.21.0(typescript@5.9.3) + specifier: 5.24.0 + version: 5.24.0(typescript@5.9.3) '@socketsecurity/registry': specifier: 'catalog:' version: 2.0.2(typescript@5.9.3) @@ -772,8 +778,8 @@ importers: packages/package-builder: dependencies: '@socketsecurity/lib': - specifier: 5.21.0 - version: 5.21.0(typescript@5.9.3) + specifier: 5.24.0 + version: 5.24.0(typescript@5.9.3) build-infra: specifier: workspace:* version: link:../build-infra @@ -2132,6 +2138,9 @@ packages: resolution: {integrity: sha512-mNe0Iigql08YupSOGv197YdHpPPr+EzDZmfCgMc7RPNaZTw5aLN01nBl6CHJOh3BGtnMIj83EeN4butBchc8Ag==} engines: {node: ^20.17.0 || >=22.9.0} + '@sinclair/typebox@0.34.49': + resolution: {integrity: sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==} + '@sindresorhus/chunkify@2.0.0': resolution: {integrity: sha512-srajPSoMTC98FETCJIeXJhJqB77IRPJSu8g907jLuuioLORHZJ3YAOY2DsP5ebrZrjOrAwjqf+Cgkg/I8TGPpw==} engines: {node: '>=18'} @@ -2151,6 +2160,7 @@ packages: '@socketaddon/iocraft@file:packages/package-builder/build/dev/out/socketaddon-iocraft': resolution: {directory: packages/package-builder/build/dev/out/socketaddon-iocraft, type: directory} + engines: {node: '>=18'} '@socketregistry/es-set-tostringtag@1.0.10': resolution: {integrity: sha512-btXmvw1JpA8WtSoXx9mTapo9NAyIDKRRzK84i48d8zc0X09M6ORfobVnHbgwhXf7CFhkRzhYrHG9dqbI9vpELQ==} @@ -2209,8 +2219,8 @@ packages: resolution: {integrity: sha512-kLKdSqi4W7SDSm5z+wYnfVRnZCVhxzbzuKcdOZSrcHoEGOT4Gl844uzoaML+f5eiQMxY+nISiETwRph/aXrIaQ==} engines: {node: 18.20.7 || ^20.18.3 || >=22.14.0} - '@socketsecurity/lib@5.21.0': - resolution: {integrity: sha512-cSqdq2kOBSuH3u8rfDhViCrN7IJPqzAvzklUYrEFhohUgJkky0+YYQ/gbSwRehZDGY8mqv+6lKGrt4OKWnNsdQ==} + '@socketsecurity/lib@5.24.0': + resolution: {integrity: sha512-4Yar8oo4N12ESoNt/i2PNf08HRABUC0OcfUfwzIF3xjq89E5VMDN+aeOtnn6Oo4Y6u3TiuZRG7NgEBZ83LQ1Lw==} engines: {node: '>=22', pnpm: '>=11.0.0-rc.0'} peerDependencies: typescript: '>=5.0.0' @@ -5652,6 +5662,8 @@ snapshots: '@sigstore/core': 3.2.0 '@sigstore/protobuf-specs': 0.5.1 + '@sinclair/typebox@0.34.49': {} + '@sindresorhus/chunkify@2.0.0': {} '@sindresorhus/df@1.0.1': {} @@ -5699,7 +5711,7 @@ snapshots: pony-cause: 2.1.11 yaml: 2.8.1 - '@socketsecurity/lib@5.21.0(typescript@5.9.3)': + '@socketsecurity/lib@5.24.0(typescript@5.9.3)': optionalDependencies: typescript: 5.9.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 3640440d3..2402f1bfd 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -41,13 +41,14 @@ catalog: '@pnpm/lockfile.fs': 1001.1.17 '@pnpm/logger': 1001.0.0 '@sentry/node': 8.0.0 + '@sinclair/typebox': 0.34.49 '@socketregistry/hyrious__bun.lockb': 1.0.19 '@socketregistry/indent-string': 1.0.14 '@socketregistry/is-interactive': 1.0.6 '@socketregistry/packageurl-js': 1.4.2 '@socketregistry/yocto-spinner': 1.0.25 '@socketsecurity/config': 3.0.1 - '@socketsecurity/lib': 5.21.0 + '@socketsecurity/lib': 5.24.0 '@socketsecurity/registry': 2.0.2 '@socketsecurity/sdk': 4.0.1 '@types/adm-zip': 0.5.7 diff --git a/scripts/build.mts b/scripts/build.mts index e383b3630..95adfb15c 100755 --- a/scripts/build.mts +++ b/scripts/build.mts @@ -32,11 +32,13 @@ import { WIN32 } from '@socketsecurity/lib/constants/platform' import { getDefaultLogger } from '@socketsecurity/lib/logger' import { spawn } from '@socketsecurity/lib/spawn' +import { CHECKPOINTS } from '../packages/build-infra/lib/constants.mts' import { PLATFORM_TARGETS, formatPlatformTarget, parsePlatformTarget, } from '../packages/build-infra/lib/platform-targets.mts' +import { runPipelineCli } from '../packages/build-infra/lib/build-pipeline.mts' const logger = getDefaultLogger() const __filename = fileURLToPath(import.meta.url) @@ -389,89 +391,85 @@ async function buildCurrentPlatformSea(): Promise<{ success: boolean }> { } /** - * Run the default smart build with caching. + * Run the default smart build — now orchestrated via the shared + * build-pipeline (same system socket-btm/ultrathink/socket-tui/sdxgen use). + * + * Stages: + * CLI build @socketsecurity/cli via pnpm --filter (the existing + * buildPackage helper handles signature check + skip-on-cached + * inside the workspace; the orchestrator's shouldRun layer + * complements with a content-hashed cache key) + * SEA build SEA binary for current platform (only when --force/--prod) + * FINALIZED verify expected outputs exist + * + * Inherits CLI flags from runPipelineCli: --force, --clean, --clean-stage, + * --from-stage, --cache-key, --prod, --dev. */ async function runSmartBuild(force: boolean): Promise { - logger.log('') - logger.log('='.repeat(60)) - logger.log(`${colors.blue('Socket CLI Build System')}`) - logger.log('='.repeat(60)) - logger.log('') - - if (force) { - logger.log(`${colors.yellow('⚠')} Force rebuild enabled (ignoring cache)`) - logger.log('') - } - - const results = [] - let totalTime = 0 - - for (const pkg of BUILD_PACKAGES) { - const startTime = Date.now() - const result = await buildPackage(pkg, force) - const duration = Date.now() - startTime - - if (!result.skipped) { - totalTime += duration - } - - results.push({ ...result, pkg }) - - if (!result.success) { - break - } - } - - // If force build and CLI built successfully, also build SEA binary for current platform. - if (force && results.every(r => r.success)) { - const startTime = Date.now() - const seaResult = await buildCurrentPlatformSea() - const duration = Date.now() - startTime - - if (!seaResult.success) { - results.push({ - success: false, - skipped: false, - pkg: { name: 'SEA Binary' }, - }) - } else { - totalTime += duration - results.push({ - success: true, - skipped: false, - pkg: { name: 'SEA Binary' }, - }) - } - } - - // Print summary. - logger.log('') - logger.log('='.repeat(60)) - logger.log(`${colors.blue('Build Summary')}`) - logger.log('='.repeat(60)) - logger.log('') - - const built = results.filter(r => r.success && !r.skipped).length - const skipped = results.filter(r => r.skipped).length - const failed = results.filter(r => !r.success).length - - logger.log(`${colors.green('Built:')} ${built}`) - logger.log(`${colors.gray('Skipped:')} ${skipped}`) - if (failed > 0) { - logger.log(`${colors.red('Failed:')} ${failed}`) - } - logger.log(`${colors.blue('Total:')} ${(totalTime / 1000).toFixed(1)}s`) - logger.log('') - - if (failed > 0) { - logger.log(`${colors.red('✗')} Build FAILED`) - logger.log('') - process.exitCode = 1 - return - } - - logger.log(`${colors.green('✓')} Build completed successfully`) - logger.log('') + // The orchestrator reads --force/--clean/... off process.argv itself; we + // only pass `force` here so the SEA stage knows whether to run. + const cliPkg = BUILD_PACKAGES[0]! + const cliOutputPath = path.join(rootDir, cliPkg.outputCheck) + + await runPipelineCli({ + packageRoot: rootDir, + packageName: 'cli', + resolvePlatformArch: async () => 'universal', + getBuildPaths: (mode: string) => ({ + buildDir: path.join(rootDir, 'build', mode), + }), + getOutputFiles: () => [cliOutputPath], + stages: [ + { + name: CHECKPOINTS.CLI, + sourcePaths: fg.sync(cliPkg.inputs, { + cwd: rootDir, + onlyFiles: true, + dot: true, + absolute: true, + }), + run: async () => { + const result = await buildPackage(cliPkg, force) + if (!result.success) { + throw new Error(`${cliPkg.name} build failed`) + } + return { + smokeTest: async () => { + if (!existsSync(cliOutputPath)) { + throw new Error(`CLI output missing: ${cliOutputPath}`) + } + }, + } + }, + }, + // SEA stage only runs when --force / --prod is set (matches the + // historical behavior of runSmartBuild — plain `pnpm build` stops + // after CLI, `pnpm build --force` also builds SEA for the current + // platform). The skip predicate runs before shouldRun(), so the + // stage is transparently absent on a normal dev build. + { + name: CHECKPOINTS.SEA, + skip: ctx => !force && ctx.buildMode !== 'prod', + run: async () => { + const seaResult = await buildCurrentPlatformSea() + if (!seaResult.success) { + throw new Error('SEA binary build failed') + } + return {} + }, + }, + { + name: CHECKPOINTS.FINALIZED, + run: async () => ({ + smokeTest: async () => { + if (!existsSync(cliOutputPath)) { + throw new Error(`CLI output missing: ${cliOutputPath}`) + } + }, + }), + }, + ], + }) } /** From 7ac5366c1f66e4d5fb8c6e889d79ab873a720ab7 Mon Sep 17 00:00:00 2001 From: jdalton Date: Wed, 22 Apr 2026 20:15:00 -0400 Subject: [PATCH 2/3] fix(build-infra): address cursor bugbot findings on build-pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - loadExternalTools() now inspects validateExternalTools's return value. The function returns { ok, errors? } per its own JSDoc; the previous call discarded it, so malformed external-tools.json silently passed through. Throw on validation failure with the formatted issue list. - validateCheckpointChain error message now enumerates Object.values (the actual checkpoint strings like 'cli', 'finalized') instead of Object.keys (the enum accessors like CLI, FINALIZED) so suggestions match what callers actually pass. - runPipelineCli no longer logs the thrown error before re-throwing — callers already have a top-level catch that formats the failure, so we were showing the same message twice. Exit code is still set here so a consumer that forgets the catch still fails the process. Reported on PR #1265. --- packages/build-infra/lib/build-pipeline.mts | 14 ++++++++++++-- packages/build-infra/lib/constants.mts | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/build-infra/lib/build-pipeline.mts b/packages/build-infra/lib/build-pipeline.mts index 96caed3df..b8fe0c9d5 100644 --- a/packages/build-infra/lib/build-pipeline.mts +++ b/packages/build-infra/lib/build-pipeline.mts @@ -150,7 +150,15 @@ async function loadExternalTools(packageRoot) { if (!data) { return { versions: {}, rawHash: '' } } - validateExternalTools(data) + const validated = validateExternalTools(data) + if (!validated.ok) { + const details = validated.errors + .map(e => ` ${e.path}: ${e.message}`) + .join('\n') + throw new Error( + `Invalid external-tools.json at ${filePath}:\n${details}`, + ) + } const versions = {} for (const [tool, meta] of Object.entries(data.tools ?? {})) { versions[tool] = meta?.version ?? '' @@ -475,7 +483,9 @@ export async function runPipelineCli(options) { try { await runPipeline(options) } catch (e) { - logger.error(errorMessage(e)) + // Set exit code and rethrow so the caller's top-level handler is the + // single place that formats/logs the failure. Logging here AND in the + // caller's catch shows the same error twice. process.exitCode = 1 throw e } diff --git a/packages/build-infra/lib/constants.mts b/packages/build-infra/lib/constants.mts index 33bc9ec70..0cc473c8e 100644 --- a/packages/build-infra/lib/constants.mts +++ b/packages/build-infra/lib/constants.mts @@ -65,7 +65,7 @@ export function validateCheckpointChain( if (invalid.length) { throw new Error( `${packageName}: Invalid checkpoint names in chain: ${invalid.join(', ')}. ` + - `Valid: ${Object.keys(CHECKPOINTS).join(', ')}`, + `Valid: ${Object.values(CHECKPOINTS).join(', ')}`, ) } const seen = new Set() From 3eb36bfc738441c74e42d9cdaf885345eb0e039a Mon Sep 17 00:00:00 2001 From: jdalton Date: Wed, 22 Apr 2026 21:24:56 -0400 Subject: [PATCH 3/3] fix(build): SEA stage must hash CLI output into its cache key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor bugbot flagged the SEA stage as missing sourcePaths: its checkpoint hash only covered external-tools.json + root package.json (the two package-wide inputs runStage always adds). If CLI rebuilt but those files didn't change, shouldRun would find the SEA checkpoint hash unchanged and skip SEA — leaving a stale binary built from the previous dist/index.js. Add cliOutputPath to the SEA stage's sourcePaths so any CLI rebuild invalidates the SEA checkpoint and forces a rebuild of the binary. The other two findings on this PR (validateCheckpointChain wording, runPipelineCli double-log) were cursor re-flagging issues already fixed in commit 7ac5366c1 — current code uses Object.values and no longer logs in runPipelineCli. --- scripts/build.mts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/build.mts b/scripts/build.mts index 95adfb15c..68597d3a2 100755 --- a/scripts/build.mts +++ b/scripts/build.mts @@ -450,6 +450,11 @@ async function runSmartBuild(force: boolean): Promise { { name: CHECKPOINTS.SEA, skip: ctx => !force && ctx.buildMode !== 'prod', + // Hash the CLI output into this stage's cache key. Without it, + // shouldRun() only sees external-tools.json + package.json, so a + // CLI rebuild that leaves those files untouched would skip SEA + // and leave a stale binary built against the previous dist/index.js. + sourcePaths: [cliOutputPath], run: async () => { const seaResult = await buildCurrentPlatformSea() if (!seaResult.success) {