diff --git a/packages/pnpm-policy/README.md b/packages/pnpm-policy/README.md index 441bf06..eee463e 100644 --- a/packages/pnpm-policy/README.md +++ b/packages/pnpm-policy/README.md @@ -153,6 +153,33 @@ That becomes pnpm's `allowBuilds` map (pnpm ≥ 10.16), with the reasons as inli Unlike the release-age exemptions, this list is **not** derived from anything: a package that runs install scripts is a deliberate trust decision, whoever published it. +## Understanding what you depend on + +Deciding what to exempt means deciding which *projects* you trust, but npm only offers accounts — and an account is as wide as everything its owner will ever publish. The person maintaining a library you want may also co-maintain something enormous you did not mean to exempt. + +`origins` answers the question npm does not: group the packages a workspace resolves by the repository they publish from. + +```bash +pnpm-policy origins # every resolved package, grouped by repo owner +pnpm-policy origins --from postgraphile # only the subtree that one dependency dragged in +pnpm-policy origins --owner acme # just that owner's packages +pnpm-policy origins --owner acme --out acme.inventory.json # written as an inventory +``` + +``` +$ pnpm-policy origins --from postgraphile +radix-ui (29) + (20) +graphile (15) +graphql (8) +``` + +`--from` reads the lockfile's dependency graph and walks it, so you see what a single decision actually pulled in rather than surveying everything at once. Transitive dependencies are included, because those are the ones an exemption list forgets. + +`--owner ... --out ...` writes the result as an inventory, ready to pass to `inventory:`. It emits **names only** — no `maintainers`, no scope globs — because the point is a reviewed list, and a glob would re-widen it to whatever gets published into that scope next. + +The repository field is self-reported, so this is a proxy for provenance, not proof of it. It answers "which project is this package from", not "is this package safe". + ## The inventory `pnpm-policy inventory` queries `registry.npmjs.org` for `maintainer:`, paginates, and writes: diff --git a/packages/pnpm-policy/__tests__/origins.test.ts b/packages/pnpm-policy/__tests__/origins.test.ts new file mode 100644 index 0000000..1114c1d --- /dev/null +++ b/packages/pnpm-policy/__tests__/origins.test.ts @@ -0,0 +1,140 @@ +import { mkdtempSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import { + groupByOwner, + namesFromOwners, + packageOrigins, + reachableFrom, + readLockfileGraph, + repositorySlug +} from '../src'; + +describe('repositorySlug', () => { + it('handles the shapes a repository field actually takes', () => { + expect(repositorySlug('git+https://github.com/graphile/crystal.git')).toBe('graphile/crystal'); + expect(repositorySlug('https://github.com/graphile/crystal')).toBe('graphile/crystal'); + expect(repositorySlug('git@github.com:graphile/crystal.git')).toBe('graphile/crystal'); + expect(repositorySlug('ssh://git@github.com/graphile/crystal.git')).toBe('graphile/crystal'); + expect(repositorySlug('graphile/crystal')).toBe('graphile/crystal'); + }); + + it('lowercases, so owner comparisons are not case-sensitive', () => { + expect(repositorySlug('https://github.com/GraphQL/graphql-js')).toBe('graphql/graphql-js'); + }); + + it('is undefined when there is nothing usable', () => { + expect(repositorySlug(undefined)).toBeUndefined(); + expect(repositorySlug('not a url')).toBeUndefined(); + }); +}); + +const LOCKFILE = `lockfileVersion: '9.0' + +importers: + .: + dependencies: + postgraphile: + specifier: ^5.0.0 + version: 5.0.0 + devDependencies: + jest: + specifier: ^30.0.0 + version: 30.0.0 + +snapshots: + postgraphile@5.0.0: + dependencies: + grafast: 1.0.0 + graphql: 16.0.0 + grafast@1.0.0: + dependencies: + graphql: 16.0.0 + tamedevil: 1.0.0 + tamedevil@1.0.0: {} + graphql@16.0.0: {} + jest@30.0.0: + dependencies: + chalk: 5.0.0 + chalk@5.0.0: {} +`; + +function lockfile(): string { + const dir = mkdtempSync(join(tmpdir(), 'pnpm-policy-graph-')); + writeFileSync(join(dir, 'pnpm-lock.yaml'), LOCKFILE); + return join(dir, 'pnpm-lock.yaml'); +} + +describe('readLockfileGraph', () => { + it('reads direct dependencies as roots, dev included', () => { + const graph = readLockfileGraph(lockfile()); + expect([...graph.roots].sort()).toEqual(['jest', 'postgraphile']); + }); + + it('maps each package to what it depends on', () => { + const graph = readLockfileGraph(lockfile()); + expect([...(graph.edges.get('postgraphile') ?? [])].sort()).toEqual(['grafast', 'graphql']); + }); +}); + +describe('reachableFrom', () => { + it('returns the subtree under a dependency, not the whole lockfile', () => { + const graph = readLockfileGraph(lockfile()); + const under = reachableFrom(graph, ['postgraphile']); + expect([...under].sort()).toEqual(['grafast', 'graphql', 'postgraphile', 'tamedevil']); + // jest is a root too, but nothing under postgraphile pulls it in. + expect(under.has('jest')).toBe(false); + expect(under.has('chalk')).toBe(false); + }); + + it('terminates on a cycle', () => { + const graph = { + roots: new Set(['a']), + edges: new Map([ + ['a', new Set(['b'])], + ['b', new Set(['a'])] + ]) + }; + expect([...reachableFrom(graph, ['a'])].sort()).toEqual(['a', 'b']); + }); +}); + +describe('packageOrigins', () => { + const packuments: Record = { + grafast: { repository: { url: 'git+https://github.com/graphile/crystal.git' } }, + graphql: { repository: { url: 'git+https://github.com/graphql/graphql-js.git' } }, + // repository only on the latest version, as older publishes sometimes do + ruru: { + 'dist-tags': { latest: '2.0.0' }, + versions: { '2.0.0': { repository: 'https://github.com/graphile/crystal' } } + } + }; + + const stub = (async (url: string) => { + const name = decodeURIComponent(url.split('/').pop() as string); + const body = packuments[name]; + return body + ? { ok: true, json: async () => body } + : { ok: false, status: 404, json: async () => ({}) }; + }) as unknown as typeof fetch; + + it('resolves owners, including a repository found only on the latest version', async () => { + const origins = await packageOrigins(['grafast', 'graphql', 'ruru'], { fetchImpl: stub }); + expect(origins.map((o) => o.owner)).toEqual(['graphile', 'graphql', 'graphile']); + }); + + it('reports an unknown package instead of aborting the survey', async () => { + const origins = await packageOrigins(['grafast', 'nope'], { fetchImpl: stub }); + expect(origins).toHaveLength(2); + expect(origins[1]).toEqual({ name: 'nope' }); + }); + + it('groups by owner and filters to the owners asked for', async () => { + const origins = await packageOrigins(['grafast', 'graphql', 'ruru'], { fetchImpl: stub }); + expect(groupByOwner(origins).get('graphile')).toEqual(['grafast', 'ruru']); + // The point of the whole exercise: graphql is a different project. + expect(namesFromOwners(origins, ['graphile'])).toEqual(['grafast', 'ruru']); + expect(namesFromOwners(origins, ['graphile'])).not.toContain('graphql'); + }); +}); diff --git a/packages/pnpm-policy/src/cli.ts b/packages/pnpm-policy/src/cli.ts index e50334c..85e9f7f 100644 --- a/packages/pnpm-policy/src/cli.ts +++ b/packages/pnpm-policy/src/cli.ts @@ -6,7 +6,10 @@ import { findConfig, loadConfig } from './config'; import { formatDuration } from './duration'; import { PolicyError } from './errors'; import { check, generate } from './generate'; +import { readWorkspaceGraph, reachableFrom } from './graph'; import { buildInventory, writeInventory } from './inventory'; +import { readWorkspacePackages } from './lockfile'; +import { groupByOwner, namesFromOwners, packageOrigins } from './origins'; import type { BuildsKey } from './policy'; const USAGE = `pnpm-policy — pnpm supply-chain policy for npm maintainers @@ -19,6 +22,7 @@ Commands: inventory Query npm for what your maintainers publish, and write the export generate Patch the policy into pnpm-workspace.yaml check Fail if the workspace file drifted or a waiver expired + origins Group this workspace's dependencies by the repository they publish from Options: --cwd Workspace root (default: current directory) @@ -28,6 +32,8 @@ Options: --no-intersect Emit every first-party name, not just the ones this workspace resolves --verify-scopes inventory: also glob a scope the registry shows nobody else publishing into (best-effort — npm's search index is incomplete) + --owner origins: keep only packages published from this repo owner (repeatable) + --from origins: limit to the subtree under this dependency (repeatable) --registry inventory: registry to query (default: https://registry.npmjs.org) --throttle inventory: pause between registry requests (default: 1000) --json Print machine-readable output @@ -83,6 +89,8 @@ interface Parsed { verifyScopes: boolean; registry?: string; throttle?: number; + owners: string[]; + from: string[]; json: boolean; quiet: boolean; help: boolean; @@ -91,6 +99,8 @@ interface Parsed { export function parseArgs(argv: string[]): Parsed { const parsed: Parsed = { + owners: [], + from: [], verifyScopes: false, json: false, quiet: false, @@ -146,6 +156,12 @@ export function parseArgs(argv: string[]): Parsed { case '--verify-scopes': parsed.verifyScopes = true; break; + case '--owner': + parsed.owners.push(next()); + break; + case '--from': + parsed.from.push(next()); + break; case '--registry': parsed.registry = next(); break; @@ -197,6 +213,88 @@ function runInit(parsed: Parsed): number { return 0; } +/** + * Group a workspace's dependencies by the repository they publish from. + * + * The question this answers is "which projects am I actually depending on", + * which is the one worth asking before deciding what to exempt from a release-age + * quarantine. Grouping by repository rather than by npm account matters: an + * account is as wide as everything its owner will ever publish, and the owner of + * a library you want may also co-maintain something far larger. + * + * With --from, only the subtree under those dependencies is considered, so you + * can ask what one decision dragged in rather than surveying the whole lockfile. + * With --owner, the output narrows to those owners and can be written straight + * out as an inventory. + */ +async function runOrigins(parsed: Parsed): Promise { + const workspaceDir = parsed.cwd ?? process.cwd(); + + let names: Set; + if (parsed.from.length) { + const graph = readWorkspaceGraph(workspaceDir); + const missing = parsed.from.filter((name) => !graph.edges.has(name) && !graph.roots.has(name)); + if (missing.length) { + console.error(`Not in this lockfile: ${missing.join(', ')}`); + return 1; + } + names = reachableFrom(graph, parsed.from); + } else { + names = readWorkspacePackages(workspaceDir); + } + + if (!parsed.quiet) { + const scope = parsed.from.length ? `under ${parsed.from.join(', ')}` : 'in this workspace'; + console.error(`Resolving repositories for ${names.size} package(s) ${scope}...`); + } + + const origins = await packageOrigins(names, { + registry: parsed.registry, + throttleMs: parsed.throttle, + onPackage: parsed.quiet + ? undefined + : (name, index, total) => { + if (index % 25 === 0) console.error(` ${index}/${total}`); + } + }); + + if (parsed.owners.length) { + const matched = namesFromOwners(origins, parsed.owners); + + if (parsed.out) { + // Deliberately no maintainers and no scopes: a list derived this way is a + // reviewed set of names, and a scope glob would re-widen it to whatever + // gets published into that scope next. + writeInventory(resolve(parsed.out), { + generatedAt: new Date().toISOString(), + maintainers: [], + scopes: [], + packages: matched + }); + if (!parsed.quiet) { + console.error(`Wrote ${matched.length} package(s) to ${parsed.out}`); + } + return 0; + } + + console.log(parsed.json ? JSON.stringify(matched, null, 2) : matched.join('\n')); + return 0; + } + + const grouped = [...groupByOwner(origins)].sort((a, b) => b[1].length - a[1].length); + + if (parsed.json) { + console.log(JSON.stringify(Object.fromEntries(grouped), null, 2)); + return 0; + } + + for (const [owner, packages] of grouped) { + console.log(`${owner || ''} (${packages.length})`); + for (const name of packages) console.log(` ${name}`); + } + return 0; +} + async function runInventory(parsed: Parsed): Promise { const { file: configFile, config } = loadConfig( parsed.config ?? parsed.cwd ?? process.cwd() @@ -334,6 +432,8 @@ export async function run(argv: string[] = process.argv.slice(2)): Promise | undefined; + +interface ImporterShape { + dependencies?: DependencyBlock; + devDependencies?: DependencyBlock; + optionalDependencies?: DependencyBlock; +} + +interface SnapshotShape { + dependencies?: Record; + optionalDependencies?: Record; +} + +interface LockfileShape { + importers?: Record; + snapshots?: Record; + packages?: Record; +} + +export interface DependencyGraph { + /** Everything the workspace's own package.json files ask for directly. */ + roots: Set; + /** package name -> the names it depends on. */ + edges: Map>; +} + +function addEdges(edges: Map>, from: string, to: Iterable): void { + const existing = edges.get(from) ?? new Set(); + for (const name of to) existing.add(name); + edges.set(from, existing); +} + +/** Read a pnpm lockfile into a name-keyed dependency graph. */ +export function readLockfileGraph(lockfilePath: string): DependencyGraph { + if (!existsSync(lockfilePath)) { + throw new PolicyError(`No ${LOCKFILE_NAME} at ${lockfilePath}. Run pnpm install first.`); + } + + const lockfile = parseYaml(readFileSync(lockfilePath, 'utf-8')) as LockfileShape | null; + const roots = new Set(); + const edges = new Map>(); + + for (const importer of Object.values(lockfile?.importers ?? {})) { + for (const block of [ + importer.dependencies, + importer.devDependencies, + importer.optionalDependencies + ]) { + for (const name of Object.keys(block ?? {})) roots.add(name); + } + } + + for (const [key, snapshot] of Object.entries(lockfile?.snapshots ?? {})) { + const from = packageNameFromLockKey(key); + if (!from) continue; + const to = [ + ...Object.keys(snapshot?.dependencies ?? {}), + ...Object.keys(snapshot?.optionalDependencies ?? {}) + ]; + addEdges(edges, from, to); + } + + return { roots, edges }; +} + +/** + * Every package reachable from `from`, including the starting names themselves. + * + * Cycles are common in a real tree, so visited names are never re-expanded. + */ +export function reachableFrom(graph: DependencyGraph, from: Iterable): Set { + const seen = new Set(); + const queue = [...from]; + + while (queue.length) { + const name = queue.pop() as string; + if (seen.has(name)) continue; + seen.add(name); + for (const next of graph.edges.get(name) ?? []) { + if (!seen.has(next)) queue.push(next); + } + } + + return seen; +} + +/** Convenience: the graph for the lockfile beside a workspace root. */ +export function readWorkspaceGraph(workspaceDir: string): DependencyGraph { + return readLockfileGraph(join(workspaceDir, LOCKFILE_NAME)); +} diff --git a/packages/pnpm-policy/src/index.ts b/packages/pnpm-policy/src/index.ts index 9aa6c70..53b75dc 100644 --- a/packages/pnpm-policy/src/index.ts +++ b/packages/pnpm-policy/src/index.ts @@ -11,6 +11,10 @@ export { formatDuration, parseDuration } from './duration'; export { PolicyError } from './errors'; export type { CheckResult, GenerateResult, RunOptions } from './generate'; export { check, generate } from './generate'; +export type { DependencyGraph } from './graph'; +export { readLockfileGraph, readWorkspaceGraph, reachableFrom } from './graph'; +export type { PackageOrigin } from './origins'; +export { groupByOwner, namesFromOwners, packageOrigins, repositorySlug } from './origins'; export type { BuildInventoryOptions } from './inventory'; export { buildInventory, diff --git a/packages/pnpm-policy/src/origins.ts b/packages/pnpm-policy/src/origins.ts new file mode 100644 index 0000000..267eaee --- /dev/null +++ b/packages/pnpm-policy/src/origins.ts @@ -0,0 +1,136 @@ +/** + * Where a package actually comes from. + * + * Deciding to trust an upstream means deciding to trust a *project*, but npm + * only offers accounts, and an account is as wide as everything its owner will + * ever publish. Someone who maintains the library you want may also co-maintain + * something enormous you did not mean to exempt. + * + * The repository a package publishes from is a much closer proxy for "the + * project", and it is checkable: `npm view repository.url`. This groups a + * set of packages by that field so a trust list can be derived from it rather + * than from an account name. + * + * It is a proxy, not proof — repository metadata is self-reported, and a + * compromised publish can claim anything. It answers "which project is this + * package from", not "is this package safe". + */ + +import { DEFAULT_REGISTRY, RegistryOptions } from './registry'; + +export interface PackageOrigin { + name: string; + /** Raw `repository.url` as published, when present. */ + repository?: string; + /** `owner/repo` for a recognised host, lowercased. */ + slug?: string; + /** The owner half of `slug` — a GitHub org or user. */ + owner?: string; +} + +const sleep = (ms: number): Promise => new Promise((done) => setTimeout(done, ms)); + +/** + * Pull `owner/repo` out of the many shapes a repository field takes: + * `git+https://github.com/a/b.git`, `git@github.com:a/b.git`, `https://github.com/a/b`, + * or the shorthand `a/b`. + */ +export function repositorySlug(url: string | undefined): string | undefined { + if (!url) return undefined; + + const cleaned = url + .replace(/^git\+/, '') + .replace(/\.git$/, '') + .replace(/^git@([^:]+):/, 'https://$1/') + .replace(/^ssh:\/\/git@/, 'https://'); + + const hosted = cleaned.match(/^https?:\/\/[^/]+\/([^/]+)\/([^/#?]+)/); + if (hosted) return `${hosted[1]}/${hosted[2]}`.toLowerCase(); + + const shorthand = cleaned.match(/^([\w.-]+)\/([\w.-]+)$/); + if (shorthand) return `${shorthand[1]}/${shorthand[2]}`.toLowerCase(); + + return undefined; +} + +interface Packument { + repository?: string | { url?: string }; + versions?: Record; + 'dist-tags'?: { latest?: string }; +} + +function repositoryUrl(packument: Packument): string | undefined { + const pick = (value: Packument['repository']): string | undefined => + typeof value === 'string' ? value : value?.url; + + // Prefer the top level; fall back to the latest version, which is where older + // publishes sometimes put it. + const top = pick(packument.repository); + if (top) return top; + + const latest = packument['dist-tags']?.latest; + return latest ? pick(packument.versions?.[latest]?.repository) : undefined; +} + +/** + * Look up the repository each package publishes from. + * + * Uses the packument endpoint rather than search: search does not return + * repository metadata, and one request per package is the honest cost. Failures + * are reported as an origin with no repository rather than throwing, so one + * unpublished or renamed package cannot abort a survey of hundreds. + */ +export async function packageOrigins( + names: Iterable, + options: RegistryOptions & { onPackage?: (name: string, index: number, total: number) => void } = {} +): Promise { + const registry = options.registry ?? DEFAULT_REGISTRY; + const doFetch = options.fetchImpl ?? fetch; + const throttleMs = options.throttleMs ?? 0; + const list = [...names]; + const origins: PackageOrigin[] = []; + + for (const [index, name] of list.entries()) { + options.onPackage?.(name, index, list.length); + + try { + const response = await doFetch(`${registry}/${encodeURIComponent(name)}`); + if (!response.ok) { + origins.push({ name }); + } else { + const repository = repositoryUrl((await response.json()) as Packument); + const slug = repositorySlug(repository); + origins.push({ + name, + ...(repository ? { repository } : {}), + ...(slug ? { slug, owner: slug.split('/')[0] } : {}) + }); + } + } catch { + origins.push({ name }); + } + + if (throttleMs && index < list.length - 1) await sleep(throttleMs); + } + + return origins; +} + +/** Group origins by repository owner; packages with no usable metadata land under `''`. */ +export function groupByOwner(origins: PackageOrigin[]): Map { + const grouped = new Map(); + for (const origin of origins) { + const key = origin.owner ?? ''; + grouped.set(key, [...(grouped.get(key) ?? []), origin.name].sort()); + } + return grouped; +} + +/** The names published from any of the given owners. */ +export function namesFromOwners(origins: PackageOrigin[], owners: string[]): string[] { + const wanted = new Set(owners.map((owner) => owner.toLowerCase())); + return origins + .filter((origin) => origin.owner && wanted.has(origin.owner)) + .map((origin) => origin.name) + .sort(); +}