diff --git a/packages/code-infra/README.md b/packages/code-infra/README.md index 425a0ac0a..e3e3b3ac0 100644 --- a/packages/code-infra/README.md +++ b/packages/code-infra/README.md @@ -35,7 +35,7 @@ This is stored in the `docs` top-level directory. ### Adding and publishing new packages -Whenever news packages are added to the repo (that will get published to npm) or a private package is turned into a public one, follow the below steps before invoking the publish workflow of the previous section. +Whenever new packages are added to the repo (that will get published to npm) or a private package is turned into a public one, follow the below steps before invoking the publish workflow of the previous section. 1. Goto your repo's code base on your system, open terminal and run: diff --git a/packages/code-infra/src/cli/cmdListWorkspaces.mjs b/packages/code-infra/src/cli/cmdListWorkspaces.mjs index ad958ce21..182fd3a1e 100644 --- a/packages/code-infra/src/cli/cmdListWorkspaces.mjs +++ b/packages/code-infra/src/cli/cmdListWorkspaces.mjs @@ -15,6 +15,7 @@ import { getWorkspacePackages } from '../utils/pnpm.mjs'; * @property {boolean} [publicOnly] - Whether to filter to only public packages * @property {'json'|'path'|'name'|'publish-dir'} [output] - Output format (name, path, or json) * @property {string} [sinceRef] - Git reference to filter changes since + * @property {string[]} [filter] - Same as filtering packages with --filter in pnpm. Only include packages matching the filter. See https://pnpm.io/filtering. */ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ @@ -37,13 +38,19 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ .option('since-ref', { type: 'string', description: 'Filter packages changed since git reference', + }) + .option('filter', { + type: 'string', + array: true, + description: + 'Same as filtering packages with --filter in pnpm. Only include packages matching the filter. See https://pnpm.io/filtering.', }); }, handler: async (argv) => { - const { publicOnly = false, output = 'name', sinceRef } = argv; + const { publicOnly = false, output = 'name', sinceRef, filter = [] } = argv; // Get packages using our helper function - const packages = await getWorkspacePackages({ sinceRef, publicOnly }); + const packages = await getWorkspacePackages({ sinceRef, publicOnly, filter }); if (output === 'json') { // Serialize packages to JSON diff --git a/packages/code-infra/src/cli/cmdNetlifyIgnore.mjs b/packages/code-infra/src/cli/cmdNetlifyIgnore.mjs index ea797d61e..d685889b6 100644 --- a/packages/code-infra/src/cli/cmdNetlifyIgnore.mjs +++ b/packages/code-infra/src/cli/cmdNetlifyIgnore.mjs @@ -12,93 +12,7 @@ import * as fs from 'node:fs/promises'; import * as path from 'node:path'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; import { toPosixPath } from '../utils/path.mjs'; -import { getWorkspacePackages } from '../utils/pnpm.mjs'; - -/** - * Get all workspace dependencies (direct and transitive) from a package - * @param {string} packageName - Package name - * @param {Map} workspaceMap - Map of workspace name to path - * @param {Map>>} cache - Cache of package resolution promises - * @returns {Promise>} Set of workspace package names (dependencies only, not including the package itself) - */ -async function getWorkspaceDependenciesRecursive(packageName, workspaceMap, cache) { - // Check cache first - const cached = cache.get(packageName); - if (cached) { - return cached; - } - - // Create the resolution promise - const promise = (async () => { - const packagePath = workspaceMap.get(packageName); - if (!packagePath) { - throw new Error(`Workspace "${packageName}" not found in the repository`); - } - - const packageJsonPath = path.join(packagePath, 'package.json'); - const content = await fs.readFile(packageJsonPath, 'utf8'); - const packageJson = JSON.parse(content); - - // Collect all dependency names - /** @type {Set} */ - const allDeps = new Set(); - if (packageJson.dependencies) { - Object.keys(packageJson.dependencies).forEach((dep) => allDeps.add(dep)); - } - if (packageJson.devDependencies) { - Object.keys(packageJson.devDependencies).forEach((dep) => allDeps.add(dep)); - } - if (packageJson.peerDependencies) { - Object.keys(packageJson.peerDependencies).forEach((dep) => allDeps.add(dep)); - } - - // Filter to only workspace dependencies - const workspaceDeps = Array.from(allDeps).filter((dep) => workspaceMap.has(dep)); - - // Recursively process workspace dependencies in parallel - const recursiveResults = await Promise.all( - workspaceDeps.map(async (dep) => { - return getWorkspaceDependenciesRecursive(dep, workspaceMap, cache); - }), - ); - - // Merge all results using flatMap - return new Set(recursiveResults.flatMap((result) => Array.from(result)).concat(workspaceDeps)); - })(); - - // Store in cache before returning - cache.set(packageName, promise); - - return promise; -} - -/** - * Get transitive workspace dependencies for a list of workspace names - * @param {string[]} workspaceNames - Array of workspace names - * @param {Map} workspaceMap - Map of workspace name to path - * @returns {Promise>} Set of workspace package names (including requested packages and all their dependencies) - */ -async function getTransitiveDependencies(workspaceNames, workspaceMap) { - // Shared cache for all workspace dependency resolution - const cache = new Map(); - - // Validate all workspace names exist - for (const workspaceName of workspaceNames) { - if (!workspaceMap.has(workspaceName)) { - throw new Error(`Workspace "${workspaceName}" not found in the repository`); - } - } - - // Process each requested workspace in parallel - const workspaceResults = await Promise.all( - workspaceNames.map((workspaceName) => - getWorkspaceDependenciesRecursive(workspaceName, workspaceMap, cache), - ), - ); - - // Merge all results using flatMap and add the original package names - return new Set(workspaceNames.concat(workspaceResults.flatMap((result) => Array.from(result)))); -} +import { getTransitiveDependencies, getWorkspacePackages } from '../utils/pnpm.mjs'; /** * Generate the ignore command string for netlify.toml @@ -220,7 +134,9 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ console.log(`Processing ${workspaceName}...`); // Get transitive dependencies for this specific workspace - const dependencyNames = await getTransitiveDependencies([workspaceName], workspaceMap); + const dependencyNames = await getTransitiveDependencies([workspaceName], { + workspacePathByName: workspaceMap, + }); // Convert package names to relative paths (normalize to POSIX separators for git) const relativePaths = Array.from(dependencyNames) diff --git a/packages/code-infra/src/cli/cmdPublish.mjs b/packages/code-infra/src/cli/cmdPublish.mjs index 7e2e7863e..db593c4a1 100644 --- a/packages/code-infra/src/cli/cmdPublish.mjs +++ b/packages/code-infra/src/cli/cmdPublish.mjs @@ -17,7 +17,11 @@ import * as fs from 'node:fs/promises'; import * as semver from 'semver'; import { persistentAuthStrategy } from '../utils/github.mjs'; -import { getWorkspacePackages, publishPackages } from '../utils/pnpm.mjs'; +import { + getWorkspacePackages, + publishPackages, + validatePublishDependencies, +} from '../utils/pnpm.mjs'; import { getCurrentGitSha, getRepositoryInfo } from '../utils/git.mjs'; const isCI = envCI().isCi; @@ -33,6 +37,7 @@ function getOctokit() { * @property {string} tag NPM dist tag to publish to * @property {boolean} ci Runs in CI environment * @property {string} [sha] Git SHA to use for the GitHub release workflow (local only) + * @property {string[]} [filter] Same as filtering packages with --filter in pnpm. Only publish packages matching the filter. See https://pnpm.io/filtering. */ /** @@ -260,10 +265,16 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ .option('sha', { type: 'string', description: 'Git SHA to use for the GitHub release workflow (local only)', + }) + .option('filter', { + type: 'string', + array: true, + description: + 'Same as filtering packages with --filter in pnpm. Only publish packages matching the filter. See https://pnpm.io/filtering.', }); }, handler: async (argv) => { - const { dryRun = false, githubRelease = false, tag = 'latest', sha } = argv; + const { dryRun = false, githubRelease = false, tag = 'latest', sha, filter = [] } = argv; if (isCI && !argv.ci) { console.error( @@ -290,13 +301,29 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ // Get all packages console.log('šŸ” Discovering all workspace packages...'); - const allPackages = await getWorkspacePackages({ publicOnly: true }); + const allPackages = await getWorkspacePackages({ publicOnly: true, filter }); if (allPackages.length === 0) { - console.log('āš ļø No public packages found in workspace'); + console.log( + `āš ļø No publishable packages found in workspace${filter.length > 0 ? ` matching filter "${filter.join(', ')}"` : ''}`, + ); return; } + if (filter.length > 0) { + console.log('šŸ” Validating workspace dependencies for filtered packages...'); + + const { issues } = await validatePublishDependencies(allPackages); + + if (issues.length > 0) { + throw new Error('Invalid dependencies structure of packages to be published.', { + cause: issues, + }); + } + + console.log('āœ… All workspace dependency requirements satisfied'); + } + // Get version from root package.json const version = await getReleaseVersion(); diff --git a/packages/code-infra/src/cli/cmdPublishCanary.mjs b/packages/code-infra/src/cli/cmdPublishCanary.mjs index 69484c23c..bdefbc1ee 100644 --- a/packages/code-infra/src/cli/cmdPublishCanary.mjs +++ b/packages/code-infra/src/cli/cmdPublishCanary.mjs @@ -16,10 +16,12 @@ import * as semver from 'semver'; import { getPackageVersionInfo, + getTransitiveDependencies, getWorkspacePackages, publishPackages, readPackageJson, semverMax, + validatePublishDependencies, writePackageJson, } from '../utils/pnpm.mjs'; import { getCurrentGitSha, getRepositoryInfo } from '../utils/git.mjs'; @@ -28,7 +30,7 @@ import { getCurrentGitSha, getRepositoryInfo } from '../utils/git.mjs'; * @typedef {Object} Args * @property {boolean} [dryRun] - Whether to run in dry-run mode * @property {boolean} [githubRelease] - Whether to create GitHub releases for canary packages - * @property {string[]} [package] - Only publish canary versions for specified packages (by name) + * @property {string[]} [filter] - Same as filtering packages with --filter in pnpm. Only publish packages matching the filter. See https://pnpm.io/filtering. */ const CANARY_TAG = 'canary'; @@ -119,80 +121,6 @@ function cleanupCommitMessage(message) { return `${prefix}${msg}`.trim(); } -async function getPackageToDependencyMap() { - /** - * @type {(PublicPackage & { dependencies: Record; private: boolean; })[]} - */ - const packagesWithDeps = JSON.parse( - (await $`pnpm ls -r --json --exclude-peers --only-projects --prod`).stdout, - ); - /** @type {Record} */ - const directPkgDependencies = packagesWithDeps - .filter((pkg) => !pkg.private) - .reduce((acc, pkg) => { - if (!pkg.name) { - return acc; - } - const deps = pkg.dependencies ? Object.keys(pkg.dependencies) : []; - if (!deps.length) { - return acc; - } - acc[pkg.name] = deps; - return acc; - }, /** @type {Record} */ ({})); - return directPkgDependencies; -} - -/** - * @param {Record} pkgGraph - */ -function resolveTransitiveDependencies(pkgGraph = {}) { - // Compute transitive (nested) dependencies limited to workspace packages and avoid cycles. - const workspacePkgNames = new Set(Object.keys(pkgGraph)); - const nestedMap = /** @type {Record} */ ({}); - - /** - * - * @param {string} pkgName - * @returns {string[]} - */ - const getTransitiveDeps = (pkgName) => { - /** - * @type {Set} - */ - const seen = new Set(); - const stack = (pkgGraph[pkgName] || []).slice(); - - while (stack.length) { - const dep = stack.pop(); - if (!dep || seen.has(dep)) { - continue; - } - // Only consider workspace packages for transitive expansion - if (!workspacePkgNames.has(dep)) { - // still record external deps as direct deps but don't traverse into them - seen.add(dep); - continue; - } - seen.add(dep); - const children = pkgGraph[dep] || []; - for (const c of children) { - if (!seen.has(c)) { - stack.push(c); - } - } - } - - return Array.from(seen); - }; - - for (const name of Object.keys(pkgGraph)) { - nestedMap[name] = getTransitiveDeps(name); - } - - return nestedMap; -} - /** * Prepare changelog data for packages using GitHub API * @param {PublicPackage[]} packagesToPublish - Packages that will be published @@ -225,14 +153,20 @@ async function prepareChangelogsFromGitCli(packagesToPublish, allPackages, canar } }), ); - // Second pass: check for dependency updates in other packages not part of git history - const pkgDependencies = await getPackageToDependencyMap(); - const transitiveDependencies = resolveTransitiveDependencies(pkgDependencies); + // Second pass: check for dependency updates in other packages not part of git history. + const workspacePathByName = new Map(allPackages.map((pkg) => [pkg.name, pkg.path])); + const publishedNames = new Set(packagesToPublish.map((p) => p.name)); + + const transitiveDepSets = await Promise.all( + allPackages.map((pkg) => + getTransitiveDependencies([pkg.name], { includeDev: false, workspacePathByName }), + ), + ); for (let i = 0; i < allPackages.length; i += 1) { const pkg = allPackages[i]; - const depsToPublish = (transitiveDependencies[pkg.name] ?? []).filter((dep) => - packagesToPublish.some((p) => p.name === dep), + const depsToPublish = [...transitiveDepSets[i]].filter( + (dep) => dep !== pkg.name && publishedNames.has(dep), ); if (depsToPublish.length === 0) { continue; @@ -548,14 +482,15 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ default: false, description: 'Create GitHub releases for published packages', }) - .option('package', { + .option('filter', { type: 'string', array: true, - description: 'Only publish canary versions for specified packages (by name)', + description: + 'Same as filtering packages with --filter in pnpm. Only publish packages matching the filter. See https://pnpm.io/filtering.', }); }, handler: async (argv) => { - const { dryRun = false, githubRelease = false, package: explicitPackages = [] } = argv; + const { dryRun = false, githubRelease = false, filter = [] } = argv; const options = { dryRun, githubRelease }; @@ -567,48 +502,41 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ console.log('šŸ“ GitHub releases will be created for published packages\n'); } - // Always get all packages first + // All public packages — needed by publishCanaryVersions to bump versions and update + // package.json across the entire workspace, even for packages not being published. console.log('šŸ” Discovering all workspace packages...'); - const allPackages = await getWorkspacePackages({ publicOnly: true }); + const filteredPackages = await getWorkspacePackages({ publicOnly: true, filter }); - if (allPackages.length === 0) { - console.log('āš ļø No public packages found in workspace'); + if (filteredPackages.length === 0) { + console.log( + `āš ļø No publishable packages found in workspace${filter.length > 0 ? ` matching filter "${filter.join(', ')}"` : ''}`, + ); return; } - // Validate that all workspace dependencies are explicitly passed by the user - if (explicitPackages.length > 0) { - const pkgDepMap = await getPackageToDependencyMap(); - const missingDeps = new Set(); - for (const pkg of explicitPackages) { - const deps = pkgDepMap[pkg] || []; - deps.forEach((dep) => { - if (!explicitPackages.includes(dep)) { - missingDeps.add(dep); - } + if (filter.length > 0) { + console.log('šŸ” Validating workspace dependencies for filtered packages...'); + + const { issues } = await validatePublishDependencies(filteredPackages); + + if (issues.length > 0) { + throw new Error('Invalid dependencies structure of packages to be published.', { + cause: issues, }); } - if (missingDeps.size > 0) { - throw new Error( - `Missing required workspace dependencies: - ${Array.from(missingDeps).join('\n ')} -Pass all workspace dependencies explicitly through the --package argument.`, - ); - } + + console.log('āœ… All workspace dependency requirements satisfied'); } - // Check for canary tag to determine selective publishing + // Check for canary tag to determine selective publishing. + // --filter is applied on top of sinceRef: publish only packages that have + // changed since the last canary tag AND match the filter. const canaryTag = await getLastCanaryTag(); console.log('šŸ” Checking for packages changed since canary tag...'); - let packages = canaryTag - ? await getWorkspacePackages({ sinceRef: canaryTag, publicOnly: true }) - : allPackages; - - // If user provided package list, filter to only those in packageNames - if (explicitPackages.length > 0) { - packages = packages.filter((pkg) => explicitPackages.includes(pkg.name)); - } + const packages = canaryTag + ? await getWorkspacePackages({ sinceRef: canaryTag, publicOnly: true, filter }) + : filteredPackages; console.log(`šŸ“‹ Found ${packages.length} packages(s) for canary publishing:`); packages.forEach((pkg) => { @@ -617,7 +545,7 @@ Pass all workspace dependencies explicitly through the --package argument.`, // Fetch version info for all packages in parallel console.log('\nšŸ” Fetching package version information...'); - const versionInfoPromises = allPackages.map(async (pkg) => { + const versionInfoPromises = filteredPackages.map(async (pkg) => { const versionInfo = await getPackageVersionInfo(pkg.name, pkg.version); return { packageName: pkg.name, versionInfo }; }); @@ -629,7 +557,7 @@ Pass all workspace dependencies explicitly through the --package argument.`, packageVersionInfo.set(packageName, versionInfo); } - await publishCanaryVersions(packages, allPackages, packageVersionInfo, options); + await publishCanaryVersions(packages, filteredPackages, packageVersionInfo, options); console.log('\nšŸ Publishing complete!'); }, diff --git a/packages/code-infra/src/utils/pnpm.mjs b/packages/code-infra/src/utils/pnpm.mjs index d35ff6fcf..1e15fdbd0 100644 --- a/packages/code-infra/src/utils/pnpm.mjs +++ b/packages/code-infra/src/utils/pnpm.mjs @@ -48,6 +48,7 @@ import * as semver from 'semver'; * @property {boolean} [publicOnly=false] - Whether to filter to only public packages * @property {boolean} [nonPublishedOnly=false] - Whether to filter to only non-published packages. It by default means public packages yet to be published. * @property {string} [cwd] - Current working directory to run pnpm command in + * @property {string[]} [filter] - Same as filtering packages with --filter in pnpm. Only include packages matching the filter. See https://pnpm.io/filtering. */ /** @@ -73,10 +74,15 @@ import * as semver from 'semver'; * @returns {Promise<(PrivatePackage | PublicPackage)[]>} Array of packages */ export async function getWorkspacePackages(options = {}) { - const { sinceRef = null, publicOnly = false, nonPublishedOnly = false } = options; + const { sinceRef = null, publicOnly = false, nonPublishedOnly = false, filter = [] } = options; // Build command with conditional filter const filterArg = sinceRef ? ['--filter', `...[${sinceRef}]`] : []; + if (filter.length > 0) { + filter.forEach((f) => { + filterArg.push('--filter', f); + }); + } const result = options.cwd ? await $({ cwd: options.cwd })`pnpm ls -r --json --depth -1 ${filterArg}` : await $`pnpm ls -r --json --depth -1 ${filterArg}`; @@ -175,6 +181,132 @@ export async function publishPackages(packages, options = {}) { await $({ stdio: 'inherit' })`pnpm -r publish --access public --tag=${tag} ${args}`; } +/** + * @typedef {Object} GetTransitiveDependenciesOptions + * @property {Map} [workspacePathByName] - Map of workspace package name to directory path + * @property {boolean} [includeDev=true] - Whether to include devDependencies in the traversal + */ + +/** + * Get all transitive workspace dependencies for a set of packages. + * + * Traverses `dependencies`, `peerDependencies`, and optionally `devDependencies`, + * following only packages that exist in `workspacePathByName`. Results are cached + * per package so each package is read from disk at most once regardless of how many + * roots depend on it. + * + * @param {string[]} packageNames - Package names to start the traversal from + * @param {GetTransitiveDependenciesOptions} [options] + * @returns {Promise>} All reachable workspace package names, including the input packages themselves + */ +export async function getTransitiveDependencies(packageNames, options = {}) { + const { includeDev = true, workspacePathByName = new Map() } = options; + + /** @type {Map>>} */ + const cache = new Map(); + + /** + * @param {string} packageName + * @returns {Promise>} + */ + function collectDeps(packageName) { + const cached = cache.get(packageName); + if (cached) { + return cached; + } + + const promise = (async () => { + const packagePath = workspacePathByName.get(packageName); + if (!packagePath) { + throw new Error(`Workspace "${packageName}" not found`); + } + + const pkgJson = await readPackageJson(packagePath); + const allDeps = new Set([ + ...Object.keys(pkgJson.dependencies ?? {}), + ...(includeDev ? Object.keys(pkgJson.devDependencies ?? {}) : []), + ...Object.keys(pkgJson.peerDependencies ?? {}), + ]); + const workspaceDeps = [...allDeps].filter((dep) => workspacePathByName.has(dep)); + + const recursiveResults = await Promise.all(workspaceDeps.map(collectDeps)); + return new Set([...workspaceDeps, ...recursiveResults.flatMap((s) => [...s])]); + })(); + + cache.set(packageName, promise); + return promise; + } + + for (const name of packageNames) { + if (!workspacePathByName.has(name)) { + throw new Error(`Workspace "${name}" not found`); + } + } + + const results = await Promise.all(packageNames.map(collectDeps)); + return new Set([...packageNames, ...results.flatMap((s) => [...s])]); +} + +/** + * Validate that a set of packages covers all of their transitive workspace dependencies, + * and that none of those dependencies are private (which would make them unpublishable). + * + * @param {PublicPackage[]} packages - The packages intended for publishing + * @returns {Promise<{issues: string[]}>} + * List of human-readable issue strings. Empty when the dependency set is valid. + */ +export async function validatePublishDependencies(packages) { + const allWorkspacePackages = await getWorkspacePackages(); + + /** @type {Map} */ + const workspacePackageByName = new Map( + allWorkspacePackages.flatMap((pkg) => (pkg.name ? [[pkg.name, pkg]] : [])), + ); + const workspacePathByName = new Map( + allWorkspacePackages.flatMap((pkg) => (pkg.name ? [[pkg.name, pkg.path]] : [])), + ); + + const publishedNames = new Set(packages.map((pkg) => pkg.name)); + const transitiveDeps = await getTransitiveDependencies( + packages.map((pkg) => pkg.name), + { includeDev: false, workspacePathByName }, + ); + + /** @type {Set} */ + const privateButRequired = new Set(); + /** @type {Set} */ + const missingFromPublish = new Set(); + + for (const depName of transitiveDeps) { + if (publishedNames.has(depName)) { + continue; + } + const workspacePkg = workspacePackageByName.get(depName); + if (workspacePkg?.isPrivate) { + privateButRequired.add(depName); + } else { + missingFromPublish.add(depName); + } + } + + /** @type {string[]} */ + const issues = []; + + if (privateButRequired.size > 0) { + issues.push( + `The following private workspace packages are required as dependencies but cannot be published: ${[...privateButRequired].join(', ')}`, + ); + } + + if (missingFromPublish.size > 0) { + issues.push( + `The following workspace packages are required as dependencies but are not included in the publish set: ${[...missingFromPublish].join(', ')}. Add them to the --filter list.`, + ); + } + + return { issues }; +} + /** * Read package.json from a directory * @param {string} packagePath - Path to package directory