From 556cf46523bdd10f8861e0be48a050869a3e139d Mon Sep 17 00:00:00 2001 From: Christopher Allford <6451942+ObliviousHarmony@users.noreply.github.com> Date: Tue, 9 Jan 2024 11:15:08 -0800 Subject: [PATCH] Added Monorepo CI Command (#43345) * Added Config Object Types * Added CI Config Parsing * Added Package Loader This is a convenience method for loading the package JSON file with support for caching the files that have already been imported. * Removed Unnecessary Package Values * Specified Node Dependencies * Added Test Environment Config Parsing * Changed Internal Config Representation For convenience it makes more sense to have a single type of Job interface that we can process. This avoids having to complicate checks when we output them. * Added Workspace Dependency Graph Using `pnpm list` we are able to build a graph of all of the workspace projects and their dependencies. This can be used to handle change cascades across a project's dependencies. * Added Changed File Detection We can use `git` to figure out what files have changed and associate them with the respective projects. This will let us identify what jobs to run based on the changes that have happened to a project. * Added Test Environment Config Parsing Tests * Added CI Config To Project Graph In the interest of making it easier to process the jobs we will store the CI config in the graph nodes. * Changed Project Graph Build Output Our usage of the graph depends a lot on how we are choosing to use it. Instead of returning all of the nodes we will return the root. * Added Change-Based Job Creation We can now marry the config, file changes, and project graph to output jobs. This supports checking the changes as well as cascade keys for test jobs. * Added Job Test Env Parsing The ideal time to parse all of the config values for the job's test environment is when creating the job. * Added Command Index With everything in place we can now add the command. In addition to that, I've fixed a few bugs that appeared when testing out the command locally. Since we aren't changing the CI config in this PR we can't easily test the actual command. * Fixed Typo --- tools/monorepo-utils/.eslintrc.js | 3 + tools/monorepo-utils/src/ci-jobs/README.md | 7 + tools/monorepo-utils/src/ci-jobs/index.ts | 29 ++ .../src/ci-jobs/lib/__tests__/config.spec.ts | 166 ++++++ .../lib/__tests__/file-changes.spec.ts | 60 +++ .../lib/__tests__/job-processing.spec.ts | 492 ++++++++++++++++++ .../lib/__tests__/package-file.spec.ts | 72 +++ .../lib/__tests__/project-graph.spec.ts | 114 ++++ .../lib/__tests__/test-environment.spec.ts | 146 ++++++ .../ci-jobs/lib/__tests__/test-package.json | 52 ++ .../ci-jobs/lib/__tests__/test-pnpm-list.json | 46 ++ .../monorepo-utils/src/ci-jobs/lib/config.ts | 323 ++++++++++++ .../src/ci-jobs/lib/file-changes.ts | 113 ++++ .../src/ci-jobs/lib/job-processing.ts | 259 +++++++++ .../src/ci-jobs/lib/package-file.ts | 34 ++ .../src/ci-jobs/lib/project-graph.ts | 118 +++++ .../src/ci-jobs/lib/test-environment.ts | 201 +++++++ tools/monorepo-utils/src/index.ts | 2 + 18 files changed, 2237 insertions(+) create mode 100644 tools/monorepo-utils/src/ci-jobs/README.md create mode 100644 tools/monorepo-utils/src/ci-jobs/index.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/__tests__/config.spec.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/__tests__/file-changes.spec.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/__tests__/job-processing.spec.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/__tests__/package-file.spec.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/__tests__/project-graph.spec.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-environment.spec.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-package.json create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-pnpm-list.json create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/config.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/file-changes.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/job-processing.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/package-file.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/project-graph.ts create mode 100644 tools/monorepo-utils/src/ci-jobs/lib/test-environment.ts diff --git a/tools/monorepo-utils/.eslintrc.js b/tools/monorepo-utils/.eslintrc.js index e4d185d8cd1..0803226b104 100644 --- a/tools/monorepo-utils/.eslintrc.js +++ b/tools/monorepo-utils/.eslintrc.js @@ -1,4 +1,7 @@ module.exports = { extends: [ 'plugin:@woocommerce/eslint-plugin/recommended' ], root: true, + rules: { + '@typescript-eslint/no-explicit-any': 'off', + }, }; diff --git a/tools/monorepo-utils/src/ci-jobs/README.md b/tools/monorepo-utils/src/ci-jobs/README.md new file mode 100644 index 00000000000..d94edb90841 --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/README.md @@ -0,0 +1,7 @@ +# CI Job Command + +A CLI command for generating the jobs needed by the `ci.yml` file. + +A CLI command for parsing CI workflow configuration from `package.json` files. + +Usage: `pnpm utils ci-jobs ` diff --git a/tools/monorepo-utils/src/ci-jobs/index.ts b/tools/monorepo-utils/src/ci-jobs/index.ts new file mode 100644 index 00000000000..cd70d814b17 --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/index.ts @@ -0,0 +1,29 @@ +/** + * External dependencies + */ +import { Command } from '@commander-js/extra-typings'; + +/** + * Internal dependencies + */ +import { Logger } from '../core/logger'; +import { buildProjectGraph } from './lib/project-graph'; +import { getFileChanges } from './lib/file-changes'; +import { createJobsForChanges } from './lib/job-processing'; + +const program = new Command( 'ci-jobs' ) + .description( + 'Generates CI workflow jobs based on the changes since the base ref.' + ) + .argument( + '', + 'Base ref to compare the current ref against for change detection.' + ) + .action( async ( baseRef: string ) => { + const projectGraph = buildProjectGraph(); + const fileChanges = getFileChanges( projectGraph, baseRef ); + const jobs = createJobsForChanges( projectGraph, fileChanges ); + Logger.notice( JSON.stringify( jobs, null, '\\t' ) ); + } ); + +export default program; diff --git a/tools/monorepo-utils/src/ci-jobs/lib/__tests__/config.spec.ts b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/config.spec.ts new file mode 100644 index 00000000000..a85cea9b67b --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/config.spec.ts @@ -0,0 +1,166 @@ +/** + * Internal dependencies + */ +import { JobType, parseCIConfig } from '../config'; + +describe( 'Config', () => { + describe( 'parseCIConfig', () => { + it( 'should parse empty config', () => { + const parsed = parseCIConfig( { name: 'foo', config: {} } ); + + expect( parsed ).toMatchObject( {} ); + } ); + + it( 'should parse lint config', () => { + const parsed = parseCIConfig( { + name: 'foo', + config: { + ci: { + lint: { + changes: '/src\\/.*\\.[jt]sx?$/', + command: 'foo', + }, + }, + }, + } ); + + expect( parsed ).toMatchObject( { + jobs: [ + { + type: JobType.Lint, + changes: [ new RegExp( '/src\\/.*\\.[jt]sx?$/' ) ], + command: 'foo', + }, + ], + } ); + } ); + + it( 'should parse lint config with changes array', () => { + const parsed = parseCIConfig( { + name: 'foo', + config: { + ci: { + lint: { + changes: [ + '/src\\/.*\\.[jt]sx?$/', + '/test\\/.*\\.[jt]sx?$/', + ], + command: 'foo', + }, + }, + }, + } ); + + expect( parsed ).toMatchObject( { + jobs: [ + { + type: JobType.Lint, + changes: [ + new RegExp( '/src\\/.*\\.[jt]sx?$/' ), + new RegExp( '/test\\/.*\\.[jt]sx?$/' ), + ], + command: 'foo', + }, + ], + } ); + } ); + + it( 'should parse test config', () => { + const parsed = parseCIConfig( { + name: 'foo', + config: { + ci: { + tests: [ + { + name: 'default', + changes: '/src\\/.*\\.[jt]sx?$/', + command: 'foo', + }, + ], + }, + }, + } ); + + expect( parsed ).toMatchObject( { + jobs: [ + { + type: JobType.Test, + name: 'default', + changes: [ new RegExp( '/src\\/.*\\.[jt]sx?$/' ) ], + command: 'foo', + }, + ], + } ); + } ); + + it( 'should parse test config with environment', () => { + const parsed = parseCIConfig( { + name: 'foo', + config: { + ci: { + tests: [ + { + name: 'default', + changes: '/src\\/.*\\.[jt]sx?$/', + command: 'foo', + testEnv: { + start: 'bar', + config: { + wpVersion: 'latest', + }, + }, + }, + ], + }, + }, + } ); + + expect( parsed ).toMatchObject( { + jobs: [ + { + type: JobType.Test, + name: 'default', + changes: [ new RegExp( '/src\\/.*\\.[jt]sx?$/' ) ], + command: 'foo', + testEnv: { + start: 'bar', + config: { + wpVersion: 'latest', + }, + }, + }, + ], + } ); + } ); + + it( 'should parse test config with cascade', () => { + const parsed = parseCIConfig( { + name: 'foo', + config: { + ci: { + tests: [ + { + name: 'default', + changes: '/src\\/.*\\.[jt]sx?$/', + command: 'foo', + cascade: 'bar', + }, + ], + }, + }, + } ); + + expect( parsed ).toMatchObject( { + jobs: [ + { + type: JobType.Test, + name: 'default', + changes: [ new RegExp( '/src\\/.*\\.[jt]sx?$/' ) ], + command: 'foo', + cascadeKeys: [ 'bar' ], + }, + ], + } ); + } ); + } ); +} ); diff --git a/tools/monorepo-utils/src/ci-jobs/lib/__tests__/file-changes.spec.ts b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/file-changes.spec.ts new file mode 100644 index 00000000000..fa62971ac0f --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/file-changes.spec.ts @@ -0,0 +1,60 @@ +/** + * External dependencies + */ +import { execSync } from 'node:child_process'; + +/** + * Internal dependencies + */ +import { getFileChanges } from '../file-changes'; + +jest.mock( 'node:child_process' ); + +describe( 'File Changes', () => { + describe( 'getFileChanges', () => { + it( 'should associate git changes with projects', () => { + jest.mocked( execSync ).mockImplementation( ( command ) => { + if ( command === 'git diff --name-only origin/trunk' ) { + return `test/project-a/package.json +foo/project-b/foo.js +bar/project-c/bar.js +baz/project-d/baz.js`; + } + + throw new Error( 'Invalid command' ); + } ); + + const fileChanges = getFileChanges( + { + name: 'project-a', + path: 'test/project-a', + dependencies: [ + { + name: 'project-b', + path: 'foo/project-b', + dependencies: [ + { + name: 'project-c', + path: 'bar/project-c', + dependencies: [], + }, + ], + }, + { + name: 'project-c', + path: 'bar/project-c', + dependencies: [], + }, + ], + }, + 'origin/trunk' + ); + + expect( fileChanges ).toMatchObject( { + 'project-a': [ 'package.json' ], + 'project-b': [ 'foo.js' ], + 'project-c': [ 'bar.js' ], + } ); + } ); + } ); +} ); diff --git a/tools/monorepo-utils/src/ci-jobs/lib/__tests__/job-processing.spec.ts b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/job-processing.spec.ts new file mode 100644 index 00000000000..b5232241059 --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/job-processing.spec.ts @@ -0,0 +1,492 @@ +/** + * Internal dependencies + */ +import { JobType } from '../config'; +import { createJobsForChanges } from '../job-processing'; +import { parseTestEnvConfig } from '../test-environment'; + +jest.mock( '../test-environment' ); + +describe( 'Job Processing', () => { + describe( 'getFileChanges', () => { + it( 'should do nothing with no CI configs', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + dependencies: [], + }, + {} + ); + + expect( jobs.lint ).toHaveLength( 0 ); + expect( jobs.test ).toHaveLength( 0 ); + } ); + + it( 'should trigger lint job for single node', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + ciConfig: { + jobs: [ + { + type: JobType.Lint, + changes: [ /test.js$/ ], + command: 'test-lint', + }, + ], + }, + dependencies: [], + }, + { + test: [ 'test.js' ], + } + ); + + expect( jobs.lint ).toHaveLength( 1 ); + expect( jobs.lint ).toContainEqual( { + projectName: 'test', + command: 'test-lint', + } ); + expect( jobs.test ).toHaveLength( 0 ); + } ); + + it( 'should not trigger lint job for single node with no changes', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + ciConfig: { + jobs: [ + { + type: JobType.Lint, + changes: [ /test.js$/ ], + command: 'test-lint', + }, + ], + }, + dependencies: [], + }, + {} + ); + + expect( jobs.lint ).toHaveLength( 0 ); + expect( jobs.test ).toHaveLength( 0 ); + } ); + + it( 'should trigger lint job for project graph', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + ciConfig: { + jobs: [ + { + type: JobType.Lint, + changes: [ /test.js$/ ], + command: 'test-lint', + }, + ], + }, + dependencies: [ + { + name: 'test-a', + path: 'test-a', + ciConfig: { + jobs: [ + { + type: JobType.Lint, + changes: [ /test-a.js$/ ], + command: 'test-lint-a', + }, + ], + }, + dependencies: [], + }, + { + name: 'test-b', + path: 'test-b', + ciConfig: { + jobs: [ + { + type: JobType.Lint, + changes: [ /test-b.js$/ ], + command: 'test-lint-b', + }, + ], + }, + dependencies: [], + }, + ], + }, + { + test: [ 'test.js' ], + 'test-a': [ 'test-ignored.js' ], + 'test-b': [ 'test-b.js' ], + } + ); + + expect( jobs.lint ).toHaveLength( 2 ); + expect( jobs.lint ).toContainEqual( { + projectName: 'test', + command: 'test-lint', + } ); + expect( jobs.lint ).toContainEqual( { + projectName: 'test-b', + command: 'test-lint-b', + } ); + expect( jobs.test ).toHaveLength( 0 ); + } ); + + it( 'should trigger lint job for project graph with empty config parent', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + dependencies: [ + { + name: 'test-a', + path: 'test-a', + ciConfig: { + jobs: [ + { + type: JobType.Lint, + changes: [ /test-a.js$/ ], + command: 'test-lint-a', + }, + ], + }, + dependencies: [], + }, + { + name: 'test-b', + path: 'test-b', + ciConfig: { + jobs: [ + { + type: JobType.Lint, + changes: [ /test-b.js$/ ], + command: 'test-lint-b', + }, + ], + }, + dependencies: [], + }, + ], + }, + { + test: [ 'test.js' ], + 'test-a': [ 'test-a.js' ], + 'test-b': [ 'test-b.js' ], + } + ); + + expect( jobs.lint ).toHaveLength( 2 ); + expect( jobs.lint ).toContainEqual( { + projectName: 'test-a', + command: 'test-lint-a', + } ); + expect( jobs.lint ).toContainEqual( { + projectName: 'test-b', + command: 'test-lint-b', + } ); + expect( jobs.test ).toHaveLength( 0 ); + } ); + + it( 'should trigger test job for single node', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default', + changes: [ /test.js$/ ], + command: 'test-cmd', + }, + ], + }, + dependencies: [], + }, + { + test: [ 'test.js' ], + } + ); + + expect( jobs.lint ).toHaveLength( 0 ); + expect( jobs.test ).toHaveLength( 1 ); + expect( jobs.test ).toContainEqual( { + projectName: 'test', + name: 'Default', + command: 'test-cmd', + } ); + } ); + + it( 'should not trigger test job for single node with no changes', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default', + changes: [ /test.js$/ ], + command: 'test-cmd', + }, + ], + }, + dependencies: [], + }, + {} + ); + + expect( jobs.lint ).toHaveLength( 0 ); + expect( jobs.test ).toHaveLength( 0 ); + } ); + + it( 'should trigger test job for project graph', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default', + changes: [ /test.js$/ ], + command: 'test-cmd', + }, + ], + }, + dependencies: [ + { + name: 'test-a', + path: 'test-a', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default A', + changes: [ /test-b.js$/ ], + command: 'test-cmd-a', + }, + ], + }, + dependencies: [], + }, + { + name: 'test-b', + path: 'test-b', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default B', + changes: [ /test-b.js$/ ], + command: 'test-cmd-b', + }, + ], + }, + dependencies: [], + }, + ], + }, + { + test: [ 'test.js' ], + 'test-a': [ 'test-ignored.js' ], + 'test-b': [ 'test-b.js' ], + } + ); + + expect( jobs.lint ).toHaveLength( 0 ); + expect( jobs.test ).toHaveLength( 2 ); + expect( jobs.test ).toContainEqual( { + projectName: 'test', + name: 'Default', + command: 'test-cmd', + } ); + expect( jobs.test ).toContainEqual( { + projectName: 'test-b', + name: 'Default B', + command: 'test-cmd-b', + } ); + } ); + + it( 'should trigger test job for dependent without changes when dependency has matching cascade key', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default', + changes: [ /test.js$/ ], + command: 'test-cmd', + cascadeKeys: [ 'test' ], + }, + ], + }, + dependencies: [ + { + name: 'test-a', + path: 'test-a', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default A', + changes: [ /test-a.js$/ ], + command: 'test-cmd-a', + cascadeKeys: [ 'test-a', 'test' ], + }, + ], + }, + dependencies: [], + }, + ], + }, + { + 'test-a': [ 'test-a.js' ], + } + ); + + expect( jobs.lint ).toHaveLength( 0 ); + expect( jobs.test ).toHaveLength( 2 ); + expect( jobs.test ).toContainEqual( { + projectName: 'test', + name: 'Default', + command: 'test-cmd', + } ); + expect( jobs.test ).toContainEqual( { + projectName: 'test-a', + name: 'Default A', + command: 'test-cmd-a', + } ); + } ); + + it( 'should isolate dependency cascade keys to prevent cross-dependency matching', async () => { + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default', + changes: [ /test.js$/ ], + command: 'test-cmd', + cascadeKeys: [ 'test' ], + }, + ], + }, + dependencies: [ + { + name: 'test-a', + path: 'test-a', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default A', + changes: [ /test-a.js$/ ], + command: 'test-cmd-a', + cascadeKeys: [ 'test-a', 'test' ], + }, + ], + }, + dependencies: [], + }, + { + name: 'test-b', + path: 'test-b', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default B', + changes: [ /test-b.js$/ ], + command: 'test-cmd-b', + cascadeKeys: [ 'test-b', 'test' ], + }, + ], + }, + dependencies: [], + }, + ], + }, + { + 'test-a': [ 'test-a.js' ], + } + ); + + expect( jobs.lint ).toHaveLength( 0 ); + expect( jobs.test ).toHaveLength( 2 ); + expect( jobs.test ).toContainEqual( { + projectName: 'test', + name: 'Default', + command: 'test-cmd', + } ); + expect( jobs.test ).toContainEqual( { + projectName: 'test-a', + name: 'Default A', + command: 'test-cmd-a', + } ); + } ); + + it( 'should trigger test job for single node and parse test environment config', async () => { + jest.mocked( parseTestEnvConfig ).mockResolvedValue( { + WP_ENV_CORE: 'https://wordpress.org/latest.zip', + } ); + + const jobs = await createJobsForChanges( + { + name: 'test', + path: 'test', + ciConfig: { + jobs: [ + { + type: JobType.Test, + name: 'Default', + changes: [ /test.js$/ ], + command: 'test-cmd', + testEnv: { + start: 'test-start', + config: { + wpVersion: 'latest', + }, + }, + }, + ], + }, + dependencies: [], + }, + { + test: [ 'test.js' ], + } + ); + + expect( jobs.lint ).toHaveLength( 0 ); + expect( jobs.test ).toHaveLength( 1 ); + expect( jobs.test ).toContainEqual( { + projectName: 'test', + name: 'Default', + command: 'test-cmd', + testEnv: { + start: 'test-start', + envVars: { + WP_ENV_CORE: 'https://wordpress.org/latest.zip', + }, + }, + } ); + } ); + } ); +} ); diff --git a/tools/monorepo-utils/src/ci-jobs/lib/__tests__/package-file.spec.ts b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/package-file.spec.ts new file mode 100644 index 00000000000..37bde75a485 --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/package-file.spec.ts @@ -0,0 +1,72 @@ +/** + * External dependencies + */ +import fs from 'node:fs'; + +/** + * Internal dependencies + */ +import { loadPackage } from '../package-file'; + +jest.mock( 'node:fs' ); + +describe( 'Package File', () => { + describe( 'loadPackage', () => { + it( "should throw for file that doesn't exist", () => { + jest.mocked( fs.readFileSync ).mockImplementation( ( path ) => { + if ( path === 'foo' ) { + throw new Error( 'ENOENT' ); + } + + return ''; + } ); + + expect( () => loadPackage( 'foo' ) ).toThrow( 'ENOENT' ); + } ); + + it( 'should load package.json', () => { + jest.mocked( fs.readFileSync ).mockImplementationOnce( ( path ) => { + if ( path === __dirname + '/test-package.json' ) { + return JSON.stringify( { + name: 'foo', + } ); + } + + throw new Error( 'ENOENT' ); + } ); + + const loadedFile = loadPackage( __dirname + '/test-package.json' ); + + expect( loadedFile ).toMatchObject( { + name: 'foo', + } ); + } ); + + it( 'should cache using normalized paths', () => { + jest.mocked( fs.readFileSync ).mockImplementationOnce( ( path ) => { + if ( path === __dirname + '/test-package.json' ) { + return JSON.stringify( { + name: 'foo', + } ); + } + + throw new Error( 'ENOENT' ); + } ); + loadPackage( __dirname + '/test-package.json' ); + + // Just throw if it's called again so that we can make sure we're using the cache. + jest.mocked( fs.readFileSync ).mockImplementationOnce( () => { + throw new Error( 'ENOENT' ); + } ); + + const cachedFile = loadPackage( + // Use a token that needs to be normalized to match the cached path. + __dirname + '/./test-package.json' + ); + + expect( cachedFile ).toMatchObject( { + name: 'foo', + } ); + } ); + } ); +} ); diff --git a/tools/monorepo-utils/src/ci-jobs/lib/__tests__/project-graph.spec.ts b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/project-graph.spec.ts new file mode 100644 index 00000000000..d08f23bc9ed --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/project-graph.spec.ts @@ -0,0 +1,114 @@ +/** + * External dependencies + */ +import { execSync } from 'node:child_process'; +import fs from 'node:fs'; + +/** + * Internal dependencies + */ +import { parseCIConfig } from '../config'; +import { loadPackage } from '../package-file'; +import { buildProjectGraph } from '../project-graph'; + +jest.mock( 'node:child_process' ); +jest.mock( '../config' ); +jest.mock( '../package-file' ); + +describe( 'Project Graph', () => { + describe( 'buildProjectGraph', () => { + it( 'should build graph from pnpm list', () => { + jest.mocked( execSync ).mockImplementation( ( command ) => { + if ( command === 'pnpm -w root' ) { + return '/test/monorepo/node_modules'; + } + + if ( command === 'pnpm -r list --only-projects --json' ) { + return fs.readFileSync( + __dirname + '/test-pnpm-list.json' + ); + } + + throw new Error( 'Invalid command' ); + } ); + + jest.mocked( loadPackage ).mockImplementation( ( path ) => { + if ( ! path.endsWith( 'package.json' ) ) { + throw new Error( 'Invalid path' ); + } + + const matches = path.match( /\/([^/]+)\/package.json$/ ); + + return { + name: matches[ 1 ], + }; + } ); + + jest.mocked( parseCIConfig ).mockImplementation( + ( packageFile ) => { + expect( packageFile ).toMatchObject( { + name: expect.stringMatching( /project-[abcd]/ ), + } ); + + return { jobs: [] }; + } + ); + + const graph = buildProjectGraph(); + + expect( loadPackage ).toHaveBeenCalled(); + expect( parseCIConfig ).toHaveBeenCalled(); + expect( graph ).toMatchObject( { + name: 'project-a', + path: 'project-a', + ciConfig: { + jobs: [], + }, + dependencies: [ + { + name: 'project-b', + path: 'project-b', + ciConfig: { + jobs: [], + }, + dependencies: [ + { + name: 'project-c', + path: 'project-c', + ciConfig: { + jobs: [], + }, + dependencies: [], + }, + ], + }, + { + name: 'project-c', + path: 'project-c', + ciConfig: { + jobs: [], + }, + dependencies: [], + }, + { + name: 'project-d', + path: 'project-d', + ciConfig: { + jobs: [], + }, + dependencies: [ + { + name: 'project-c', + path: 'project-c', + ciConfig: { + jobs: [], + }, + dependencies: [], + }, + ], + }, + ], + } ); + } ); + } ); +} ); diff --git a/tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-environment.spec.ts b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-environment.spec.ts new file mode 100644 index 00000000000..ef98eff44de --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-environment.spec.ts @@ -0,0 +1,146 @@ +/** + * External dependencies + */ +import { IncomingMessage, get } from 'node:http'; +import { Stream } from 'node:stream'; + +/** + * Internal dependencies + */ +import { parseTestEnvConfig } from '../test-environment'; + +jest.mock( 'node:http' ); + +describe( 'Test Environment', () => { + describe( 'parseTestEnvConfig', () => { + it( 'should parse empty configs', async () => { + const envVars = await parseTestEnvConfig( {} ); + + expect( envVars ).toEqual( {} ); + } ); + + describe( 'wpVersion', () => { + // We're going to mock an implementation of the request to the WordPress.org API. + // This simulates what happens when we call https.get() for it. + jest.mocked( get ).mockImplementation( ( url, callback: any ) => { + if ( + url !== 'http://api.wordpress.org/core/stable-check/1.0/' + ) { + throw new Error( 'Invalid URL' ); + } + + const getStream = new Stream(); + + // Let the consumer set up listeners for the stream. + callback( getStream as IncomingMessage ); + + const wpVersions = { + '5.9': 'insecure', + '6.0': 'insecure', + '6.0.1': 'insecure', + '6.1': 'insecure', + '6.1.1': 'insecure', + '6.1.2': 'outdated', + '6.2': 'latest', + }; + + getStream.emit( 'data', JSON.stringify( wpVersions ) ); + + getStream.emit( 'end' ); // this will trigger the promise resolve + + return jest.fn() as any; + } ); + + it( 'should parse "master" and "trunk" branches', async () => { + let envVars = await parseTestEnvConfig( { + wpVersion: 'master', + } ); + + expect( envVars ).toEqual( { + WP_ENV_CORE: 'WordPress/WordPress#master', + } ); + + envVars = await parseTestEnvConfig( { + wpVersion: 'trunk', + } ); + + expect( envVars ).toEqual( { + WP_ENV_CORE: 'WordPress/WordPress#master', + } ); + } ); + + it( 'should parse nightlies', async () => { + const envVars = await parseTestEnvConfig( { + wpVersion: 'nightly', + } ); + + expect( envVars ).toEqual( { + WP_ENV_CORE: + 'https://wordpress.org/nightly-builds/wordpress-latest.zip', + } ); + } ); + + it( 'should parse latest', async () => { + const envVars = await parseTestEnvConfig( { + wpVersion: 'latest', + } ); + + expect( envVars ).toEqual( { + WP_ENV_CORE: 'https://wordpress.org/latest.zip', + } ); + } ); + + it( 'should parse specific minor version', async () => { + const envVars = await parseTestEnvConfig( { + wpVersion: '5.9.0', + } ); + + expect( envVars ).toEqual( { + WP_ENV_CORE: 'https://wordpress.org/wordpress-5.9.zip', + } ); + } ); + + it( 'should parse specific patch version', async () => { + const envVars = await parseTestEnvConfig( { + wpVersion: '6.0.1', + } ); + + expect( envVars ).toEqual( { + WP_ENV_CORE: 'https://wordpress.org/wordpress-6.0.1.zip', + } ); + } ); + + it( 'should throw for version that does not exist', async () => { + const expectation = () => + parseTestEnvConfig( { + wpVersion: '1.0', + } ); + + expect( expectation ).rejects.toThrowError( + /Failed to parse WP version/ + ); + } ); + + it( 'should parse latest offset', async () => { + const envVars = await parseTestEnvConfig( { + wpVersion: 'latest-1', + } ); + + expect( envVars ).toEqual( { + WP_ENV_CORE: 'https://wordpress.org/wordpress-6.1.2.zip', + } ); + } ); + + it( 'should throw for latest offset that does not exist', async () => { + const expectation = () => + parseTestEnvConfig( { + wpVersion: 'latest-10', + } ); + + expect( expectation ).rejects.toThrowError( + /Failed to parse WP version/ + ); + } ); + } ); + } ); +} ); diff --git a/tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-package.json b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-package.json new file mode 100644 index 00000000000..645c3e7647f --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-package.json @@ -0,0 +1,52 @@ +{ + "name": "test-package-json", + "config": { + "ci": { + "lint": { + "changes": "src/.*\\.[jt]sx?$", + "command": "foo" + }, + "test": [ + { + "name": "Minimal", + "changes": ".*", + "command": "foo" + }, + { + "name": "Changes Array", + "changes": [ + "foo/.*", + "bar/.*" + ], + "command": "foo" + }, + { + "name": "Test Environment", + "changes": "env/.*", + "command": "bar", + "testEnv": { + "start": "foo", + "config": { + "wpVersion": "latest" + } + } + }, + { + "name": "Single Cascade", + "changes": "cascade/.*", + "command": "bar", + "cascade": "foo" + }, + { + "name": "Array Cascade", + "changes": "cascade/.*", + "command": "bar", + "cascade": [ + "foo", + "bar" + ] + } + ] + } + } +} diff --git a/tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-pnpm-list.json b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-pnpm-list.json new file mode 100644 index 00000000000..bfbabbf8fda --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/__tests__/test-pnpm-list.json @@ -0,0 +1,46 @@ +[ + { + "name": "project-a", + "path": "/test/monorepo/project-a", + "dependencies": { + "project-b": { + "from": "project-b", + "version": "link:../project-b", + "path": "/test/monorepo/project-b" + } + }, + "devDependencies": { + "project-c": { + "from": "project-c", + "version": "link:../project-c", + "path": "/test/monorepo/project-c" + } + } + }, + { + "name": "project-b", + "path": "/test/monorepo/project-b", + "dependencies": { + "project-c": { + "from": "project-c", + "version": "link:../project-c", + "path": "/test/monorepo/project-c" + } + } + }, + { + "name": "project-c", + "path": "/test/monorepo/project-c" + }, + { + "name": "project-d", + "path": "/test/monorepo/project-d", + "dependencies": { + "project-c": { + "from": "project-c", + "version": "link:../project-c", + "path": "/test/monorepo/project-c" + } + } + } +] diff --git a/tools/monorepo-utils/src/ci-jobs/lib/config.ts b/tools/monorepo-utils/src/ci-jobs/lib/config.ts new file mode 100644 index 00000000000..7c9fd0a0d16 --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/config.ts @@ -0,0 +1,323 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +/** + * Internal dependencies + */ +import { PackageJSON } from './package-file'; + +/** + * A configuration error type. + */ +export class ConfigError extends Error {} + +/** + * The type of the job. + */ +export const enum JobType { + Lint = 'lint', + Test = 'test', +} + +/** + * Parses and validates a raw change config entry. + * + * @param {string|string[]} raw The raw config to parse. + */ +function parseChangesConfig( raw: unknown ): RegExp[] { + if ( typeof raw === 'string' ) { + return [ new RegExp( raw ) ]; + } + + if ( ! Array.isArray( raw ) ) { + throw new ConfigError( + 'Changes configuration must be a string or array of strings.' + ); + } + + const changes: RegExp[] = []; + for ( const entry of raw ) { + if ( typeof entry !== 'string' ) { + throw new ConfigError( + 'Changes configuration must be a string or array of strings.' + ); + } + + changes.push( new RegExp( entry ) ); + } + return changes; +} + +/** + * The configuration of the lint job. + */ +export interface LintJobConfig { + /** + * The type of the job. + */ + type: JobType.Lint; + + /** + * The changes that should trigger this job. + */ + changes: RegExp[]; + + /** + * The linting command to run. + */ + command: string; +} + +/** + * Parses the lint job configuration. + * + * @param {Object} raw The raw config to parse. + */ +function parseLintJobConfig( raw: any ): LintJobConfig { + if ( ! raw.changes ) { + throw new ConfigError( + 'A "changes" option is required for the lint job.' + ); + } + + if ( ! raw.command || typeof raw.command !== 'string' ) { + throw new ConfigError( + 'A string "command" option is required for the lint job.' + ); + } + + return { + type: JobType.Lint, + changes: parseChangesConfig( raw.changes ), + command: raw.command, + }; +} + +/** + * The configuration vars for test environments. + */ +export interface TestEnvConfigVars { + /** + * The version of WordPress that should be used. + */ + wpVersion?: string; + + /** + * The version of PHP that should be used. + */ + phpVersion?: string; +} + +/** + * Parses the test env config vars. + * + * @param {Object} raw The raw config to parse. + */ +function parseTestEnvConfigVars( raw: any ): TestEnvConfigVars { + const config: TestEnvConfigVars = {}; + + if ( raw.wpVersion ) { + if ( typeof raw.wpVersion !== 'string' ) { + throw new ConfigError( 'The "wpVersion" option must be a string.' ); + } + + config.wpVersion = raw.wpVersion; + } + + if ( raw.phpVersion ) { + if ( typeof raw.phpVersion !== 'string' ) { + throw new ConfigError( + 'The "phpVersion" option must be a string.' + ); + } + + config.phpVersion = raw.phpVersion; + } + + return config; +} + +/** + * The configuration of a test environment. + */ +interface TestEnvConfig { + /** + * The command that should be used to start the test environment. + */ + start: string; + + /** + * Any configuration variables that should be used when building the environment. + */ + config: TestEnvConfigVars; +} + +/** + * The configuration of a test job. + */ +export interface TestJobConfig { + /** + * The type of the job. + */ + type: JobType.Test; + + /** + * The name for the job. + */ + name: string; + + /** + * The changes that should trigger this job. + */ + changes: RegExp[]; + + /** + * The test command to run. + */ + command: string; + + /** + * The configuration for the test environment if one is needed. + */ + testEnv?: TestEnvConfig; + + /** + * The key(s) to use when identifying what jobs should be triggered by a cascade. + */ + cascadeKeys?: string[]; +} + +/** + * parses the cascade config. + * + * @param {string|string[]} raw The raw config to parse. + */ +function parseTestCascade( raw: unknown ): string[] { + if ( typeof raw === 'string' ) { + return [ raw ]; + } + + if ( ! Array.isArray( raw ) ) { + throw new ConfigError( + 'Cascade configuration must be a string or array of strings.' + ); + } + + const changes: string[] = []; + for ( const entry of raw ) { + if ( typeof entry !== 'string' ) { + throw new ConfigError( + 'Cascade configuration must be a string or array of strings.' + ); + } + + changes.push( entry ); + } + return changes; +} + +/** + * Parses the test job config. + * + * @param {Object} raw The raw config to parse. + */ +function parseTestJobConfig( raw: any ): TestJobConfig { + if ( ! raw.name || typeof raw.name !== 'string' ) { + throw new ConfigError( + 'A string "name" option is required for test jobs.' + ); + } + + if ( ! raw.changes ) { + throw new ConfigError( + 'A "changes" option is required for the test jobs.' + ); + } + + if ( ! raw.command || typeof raw.command !== 'string' ) { + throw new ConfigError( + 'A string "command" option is required for the test jobs.' + ); + } + + const config: TestJobConfig = { + type: JobType.Test, + name: raw.name, + changes: parseChangesConfig( raw.changes ), + command: raw.command, + }; + + if ( raw.testEnv ) { + if ( typeof raw.testEnv !== 'object' ) { + throw new ConfigError( 'The "testEnv" option must be an object.' ); + } + + if ( ! raw.testEnv.start || typeof raw.testEnv.start !== 'string' ) { + throw new ConfigError( + 'A string "start" option is required for test environments.' + ); + } + + config.testEnv = { + start: raw.testEnv.start, + config: parseTestEnvConfigVars( raw.testEnv.config ), + }; + } + + if ( raw.cascade ) { + config.cascadeKeys = parseTestCascade( raw.cascade ); + } + + return config; +} + +/** + * The configuration of a job. + */ +type JobConfig = LintJobConfig | TestJobConfig; + +/** + * A project's CI configuration. + */ +export interface CIConfig { + /** + * The configuration for jobs in this config. + */ + jobs: JobConfig[]; +} + +/** + * Parses the raw CI config. + * + * @param {Object} raw The raw config. + */ +export function parseCIConfig( raw: PackageJSON ): CIConfig { + const config: CIConfig = { + jobs: [], + }; + + const ciConfig = raw.config?.ci; + + if ( ! ciConfig ) { + return config; + } + + if ( ciConfig.lint ) { + if ( typeof ciConfig.lint !== 'object' ) { + throw new ConfigError( 'The "lint" option must be an object.' ); + } + + config.jobs.push( parseLintJobConfig( ciConfig.lint ) ); + } + + if ( ciConfig.tests ) { + if ( ! Array.isArray( ciConfig.tests ) ) { + throw new ConfigError( 'The "tests" option must be an array.' ); + } + + for ( const rawTestConfig of ciConfig.tests ) { + config.jobs.push( parseTestJobConfig( rawTestConfig ) ); + } + } + + return config; +} diff --git a/tools/monorepo-utils/src/ci-jobs/lib/file-changes.ts b/tools/monorepo-utils/src/ci-jobs/lib/file-changes.ts new file mode 100644 index 00000000000..cfc3c761d1f --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/file-changes.ts @@ -0,0 +1,113 @@ +/** + * External dependencies + */ +import { execSync } from 'node:child_process'; + +/** + * Internal dependencies + */ +import { ProjectNode } from './project-graph'; + +/** + * A map of changed files keyed by the project name. + */ +export interface ProjectFileChanges { + [ name: string ]: string[]; +} + +/** + * Gets the project path for every project in the graph. + * + * @param {Object} graph The project graph to process. + * @return {Object} The project paths keyed by the project name. + */ +function getProjectPaths( graph: ProjectNode ): { [ name: string ]: string } { + const projectPaths: { [ name: string ]: string } = {}; + + const queue = [ graph ]; + const visited: { [ name: string ]: boolean } = {}; + while ( queue.length > 0 ) { + const node = queue.shift(); + if ( ! node ) { + continue; + } + + if ( visited[ node.name ] ) { + continue; + } + + projectPaths[ node.name ] = node.path; + + visited[ node.name ] = true; + + queue.push( ...node.dependencies ); + } + + return projectPaths; +} + +/** + * Checks the changed files and returns any that are relevant to the project. + * + * @param {string} projectPath The path to the project to get changed files for. + * @param {Array.} changedFiles The files that have changed in the repo. + * @return {Array.} The files that have changed in the project. + */ +function getChangedFilesForProject( + projectPath: string, + changedFiles: string[] +): string[] { + const projectChanges = []; + + // Find all of the files that have changed in the project. + for ( const filePath of changedFiles ) { + if ( ! filePath.startsWith( projectPath ) ) { + continue; + } + + // Track the file relative to the project. + projectChanges.push( filePath.slice( projectPath.length + 1 ) ); + } + + return projectChanges; +} + +/** + * Pulls all of the files that have changed in the project graph since the given git ref. + * + * @param {Object} projectGraph The project graph to assign changes for. + * @param {string} baseRef The git ref to compare against for changes. + * @return {Object} A map of changed files keyed by the project name. + */ +export function getFileChanges( + projectGraph: ProjectNode, + baseRef: string +): ProjectFileChanges { + const projectPaths = getProjectPaths( projectGraph ); + + // We're going to use git to figure out what files have changed. + const output = execSync( `git diff --name-only ${ baseRef }`, { + encoding: 'utf8', + } ); + const changedFilePaths = output.split( '\n' ); + + const changes: ProjectFileChanges = {}; + for ( const projectName in projectPaths ) { + // Projects with no paths have no changed files for us to identify. + if ( ! projectPaths[ projectName ] ) { + continue; + } + + const projectChanges = getChangedFilesForProject( + projectPaths[ projectName ], + changedFilePaths + ); + if ( projectChanges.length === 0 ) { + continue; + } + + changes[ projectName ] = projectChanges; + } + + return changes; +} diff --git a/tools/monorepo-utils/src/ci-jobs/lib/job-processing.ts b/tools/monorepo-utils/src/ci-jobs/lib/job-processing.ts new file mode 100644 index 00000000000..ac3c7894526 --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/job-processing.ts @@ -0,0 +1,259 @@ +/** + * Internal dependencies + */ +import { JobType, LintJobConfig, TestJobConfig } from './config'; +import { ProjectFileChanges } from './file-changes'; +import { ProjectNode } from './project-graph'; +import { TestEnvVars, parseTestEnvConfig } from './test-environment'; + +/** + * A linting job. + */ +interface LintJob { + projectName: string; + command: string; +} + +/** + * A testing job environment. + */ +interface TestJobEnv { + start: string; + envVars: TestEnvVars; +} + +/** + * A testing job. + */ +interface TestJob { + projectName: string; + name: string; + command: string; + testEnv?: TestJobEnv; +} + +/** + * All of the jobs that should be run. + */ +interface Jobs { + lint: LintJob[]; + test: TestJob[]; +} + +/** + * Checks the config against the changes and creates one if it should be run. + * + * @param {string} projectName The name of the project that the job is for. + * @param {Object} config The config object for the lint job. + * @param {Array.} changes The file changes that have occurred for the project. + * @return {Object|null} The job that should be run or null if no job should be run. + */ +function createLintJob( + projectName: string, + config: LintJobConfig, + changes: string[] +): LintJob | null { + let triggered = false; + + // Projects can configure jobs to be triggered when a + // changed file matches a path regex. + for ( const file of changes ) { + for ( const change of config.changes ) { + if ( change.test( file ) ) { + triggered = true; + break; + } + } + + if ( triggered ) { + break; + } + } + + if ( ! triggered ) { + return null; + } + + return { + projectName, + command: config.command, + }; +} + +/** + * Checks the config against the changes and creates one if it should be run. + * + * @param {string} projectName The name of the project that the job is for. + * @param {Object} config The config object for the test job. + * @param {Array.} changes The file changes that have occurred for the project. + * @param {Array.} cascadeKeys The cascade keys that have been triggered in dependencies. + * @return {Promise.} The job that should be run or null if no job should be run. + */ +async function createTestJob( + projectName: string, + config: TestJobConfig, + changes: string[], + cascadeKeys: string[] +): Promise< TestJob | null > { + let triggered = false; + + // Some jobs can be configured to trigger when a dependency has a job that + // was triggered. For example, a code change in a dependency might mean + // that code is impacted in the current project even if no files were + // actually changed in this project. + if ( + config.cascadeKeys && + config.cascadeKeys.some( ( value ) => cascadeKeys.includes( value ) ) + ) { + triggered = true; + } + + // Projects can configure jobs to be triggered when a + // changed file matches a path regex. + if ( ! triggered ) { + for ( const file of changes ) { + for ( const change of config.changes ) { + if ( change.test( file ) ) { + triggered = true; + break; + } + } + + if ( triggered ) { + break; + } + } + } + + if ( ! triggered ) { + return null; + } + + const createdJob: TestJob = { + projectName, + name: config.name, + command: config.command, + }; + + // We want to make sure that we're including the configuration for + // any test environment that the job will need in order to run. + if ( config.testEnv ) { + createdJob.testEnv = { + start: config.testEnv.start, + envVars: await parseTestEnvConfig( config.testEnv.config ), + }; + } + + return createdJob; +} + +/** + * Recursively checks the project for any jobs that should be executed and returns them. + * + * @param {Object} node The current project node to examine. + * @param {Object} changedFiles The files that have changed for the project. + * @param {Array.} cascadeKeys The cascade keys that have been triggered in dependencies. + * @return {Promise.} The jobs that have been created for the project. + */ +async function createJobsForProject( + node: ProjectNode, + changedFiles: ProjectFileChanges, + cascadeKeys: string[] +): Promise< Jobs > { + // We're going to traverse the project graph and check each node for any jobs that should be triggered. + const newJobs: Jobs = { + lint: [], + test: [], + }; + + // In order to simplify the way that cascades work we're going to recurse depth-first and check our dependencies + // for jobs before ourselves. This lets any cascade keys created in dependencies cascade to dependents. + const newCascadeKeys = []; + for ( const dependency of node.dependencies ) { + // Each dependency needs to have its own cascade keys so that they don't cross-contaminate. + const dependencyCascade = [ ...cascadeKeys ]; + + const dependencyJobs = await createJobsForProject( + dependency, + changedFiles, + dependencyCascade + ); + newJobs.lint.push( ...dependencyJobs.lint ); + newJobs.test.push( ...dependencyJobs.test ); + + // Track any new cascade keys added by the dependency. + // Since we're filtering out duplicates after the + // dependencies are checked we don't need to + // worry about their presence right now. + newCascadeKeys.push( ...dependencyCascade ); + } + + // Now that we're done looking at the dependencies we can add the cascade keys that + // they created. Make sure to avoid adding duplicates so that we don't waste time + // checking the same keys multiple times when we create the jobs. + cascadeKeys.push( + ...newCascadeKeys.filter( ( value ) => ! cascadeKeys.includes( value ) ) + ); + + // Projects that don't have any CI configuration don't have any potential jobs for us to check for. + if ( ! node.ciConfig ) { + return newJobs; + } + + for ( const jobConfig of node.ciConfig.jobs ) { + switch ( jobConfig.type ) { + case JobType.Lint: { + const created = createLintJob( + node.name, + jobConfig, + changedFiles[ node.name ] ?? [] + ); + if ( ! created ) { + break; + } + + newJobs.lint.push( created ); + break; + } + + case JobType.Test: { + const created = await createTestJob( + node.name, + jobConfig, + changedFiles[ node.name ] ?? [], + cascadeKeys + ); + if ( ! created ) { + break; + } + + newJobs.test.push( created ); + + // We need to track any cascade keys that this job is associated with so that + // dependent projects can trigger jobs with matching keys. We are expecting + // the array passed to this function to be modified by reference so this + // behavior is intentional. + if ( jobConfig.cascadeKeys ) { + cascadeKeys.push( ...jobConfig.cascadeKeys ); + } + break; + } + } + } + + return newJobs; +} + +/** + * Creates jobs to run for the given project graph and file changes. + * + * @param {Object} root The root node for the project graph. + * @param {Object} changes The file changes that have occurred. + * @return {Promise.} The jobs that should be run. + */ +export function createJobsForChanges( + root: ProjectNode, + changes: ProjectFileChanges +): Promise< Jobs > { + return createJobsForProject( root, changes, [] ); +} diff --git a/tools/monorepo-utils/src/ci-jobs/lib/package-file.ts b/tools/monorepo-utils/src/ci-jobs/lib/package-file.ts new file mode 100644 index 00000000000..68083f7434d --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/package-file.ts @@ -0,0 +1,34 @@ +/** + * External dependencies + */ +import fs from 'node:fs'; +import path from 'node:path'; + +export interface PackageJSON { + name: string; + config?: { ci?: any }; +} + +// We're going to store a cache of package files so that we don't load +// ones that we have already loaded. The key is the normalized path +// to the package file that was loaded. +const packageCache: { [ key: string ]: PackageJSON } = {}; + +/** + * Loads a package file's contents either from the cache or from the file system. + * + * @param {string} packagePath The package file to load. + * @return {Object} The package file's contents. + */ +export function loadPackage( packagePath: string ): PackageJSON { + // Use normalized paths to accomodate any path tokens. + packagePath = path.normalize( packagePath ); + if ( packageCache[ packagePath ] ) { + return packageCache[ packagePath ]; + } + + packageCache[ packagePath ] = JSON.parse( + fs.readFileSync( packagePath, 'utf8' ) + ); + return packageCache[ packagePath ]; +} diff --git a/tools/monorepo-utils/src/ci-jobs/lib/project-graph.ts b/tools/monorepo-utils/src/ci-jobs/lib/project-graph.ts new file mode 100644 index 00000000000..5947acf8d31 --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/project-graph.ts @@ -0,0 +1,118 @@ +/** + * External dependencies + */ +import { execSync } from 'node:child_process'; +import path from 'node:path'; + +/** + * Internal dependencies + */ +import { CIConfig, parseCIConfig } from './config'; +import { loadPackage } from './package-file'; + +/** + * A node in the project dependency graph. + */ +export interface ProjectNode { + name: string; + path: string; + ciConfig?: CIConfig; + dependencies: ProjectNode[]; +} + +/** + * Builds a dependency graph of all projects in the monorepo and returns the root node. + */ +export function buildProjectGraph(): ProjectNode { + // Get the root of the monorepo. + const monorepoRoot = path.join( + execSync( 'pnpm -w root', { encoding: 'utf-8' } ), + '..' + ); + + // PNPM provides us with a flat list of all projects + // in the workspace and their dependencies. + const workspace = JSON.parse( + execSync( 'pnpm -r list --only-projects --json', { encoding: 'utf-8' } ) + ); + + // Start by building an object containing all of the nodes keyed by their project name. + // This will let us link them together quickly by iterating through the list of + // dependencies and adding the applicable nodes. + const nodes: { [ name: string ]: ProjectNode } = {}; + let rootNode; + for ( const project of workspace ) { + // Use a relative path to the project so that it's easier for us to work with + const projectPath = project.path.replace( + new RegExp( + `^${ monorepoRoot.replace( /\\/g, '\\\\' ) }${ path.sep }?` + ), + '' + ); + + const packageFile = loadPackage( + path.join( project.path, 'package.json' ) + ); + + const ciConfig = parseCIConfig( packageFile ); + + const node = { + name: project.name, + path: projectPath, + ciConfig, + dependencies: [], + }; + + // The first entry that `pnpm list` returns is the workspace root. + // This will be the root node of our graph. + if ( ! rootNode ) { + rootNode = node; + } + + nodes[ project.name ] = node; + } + + // One thing to keep in mind is that, technically, our dependency graph has multiple roots. + // Each package that has no dependencies is a "root", however, for simplicity, we will + // add these root packages under the monorepo root in order to have a clean graph. + // Since the monorepo root has no CI config this won't cause any problems. + // Track this by recording all of the dependencies and removing them + // from the rootless list if they are added as a dependency. + const rootlessDependencies = workspace.map( ( project ) => project.name ); + + // Now we can scan through all of the nodes and hook them up to their respective dependency nodes. + for ( const project of workspace ) { + const node = nodes[ project.name ]; + if ( project.dependencies ) { + for ( const dependency in project.dependencies ) { + node.dependencies.push( nodes[ dependency ] ); + } + } + if ( project.devDependencies ) { + for ( const dependency in project.devDependencies ) { + node.dependencies.push( nodes[ dependency ] ); + } + } + + // Mark any dependencies that have a dependent as not being rootless. + // A rootless dependency is one that nothing depends on. + for ( const dependency of node.dependencies ) { + const index = rootlessDependencies.indexOf( dependency.name ); + if ( index > -1 ) { + rootlessDependencies.splice( index, 1 ); + } + } + } + + // Track the rootless dependencies now that we have them. + for ( const rootless of rootlessDependencies ) { + // Don't add the root node as a dependency of itself. + if ( rootless === rootNode.name ) { + continue; + } + + rootNode.dependencies.push( nodes[ rootless ] ); + } + + return rootNode; +} diff --git a/tools/monorepo-utils/src/ci-jobs/lib/test-environment.ts b/tools/monorepo-utils/src/ci-jobs/lib/test-environment.ts new file mode 100644 index 00000000000..095a4c66beb --- /dev/null +++ b/tools/monorepo-utils/src/ci-jobs/lib/test-environment.ts @@ -0,0 +1,201 @@ +/** + * External dependencies + */ +import https from 'node:http'; + +/** + * Internal dependencies + */ +import { TestEnvConfigVars } from './config'; + +/** + * The response for the WordPress.org stability check API. + */ +interface StableCheckResponse { + [ version: string ]: 'latest' | 'outdated' | 'insecure'; +} + +/** + * Gets all of the available WordPress versions and their associated stability. + * + * @return {Promise.} The response from the WordPress.org API. + */ +function getWordPressVersions(): Promise< StableCheckResponse > { + return new Promise< StableCheckResponse >( ( resolve, reject ) => { + // We're going to use the WordPress.org API to get information about available versions of WordPress. + const request = https.get( + 'http://api.wordpress.org/core/stable-check/1.0/', + ( response ) => { + // Listen for the response data. + let responseData = ''; + response.on( 'data', ( chunk ) => { + responseData += chunk; + } ); + + // Once we have the entire response we can process it. + response.on( 'end', () => + resolve( JSON.parse( responseData ) ) + ); + } + ); + + request.on( 'error', ( error ) => { + reject( error ); + } ); + } ); +} + +/** + * Uses the WordPress API to get the download URL to the latest version of an X.X version line. This + * also accepts "latest-X" to get an offset from the latest version of WordPress. + * + * @param {string} wpVersion The version of WordPress to look for. + * @return {Promise.} The precise WP version download URL. + */ +async function getPreciseWPVersionURL( wpVersion: string ): Promise< string > { + const allVersions = await getWordPressVersions(); + + // If we're requesting a "latest" offset then we need to figure out what version line we're offsetting from. + const latestSubMatch = wpVersion.match( /^latest(?:-([0-9]+))?$/i ); + if ( latestSubMatch ) { + for ( const version in allVersions ) { + if ( allVersions[ version ] !== 'latest' ) { + continue; + } + + // We don't care about the patch version because we will + // the latest version from the version line below. + const versionParts = version.match( /^([0-9]+)\.([0-9]+)/ ); + + // We're going to subtract the offset to figure out the right version. + let offset = latestSubMatch[ 1 ] + ? parseInt( latestSubMatch[ 1 ], 10 ) + : 0; + let majorVersion = parseInt( versionParts[ 1 ], 10 ); + let minorVersion = parseInt( versionParts[ 2 ], 10 ); + while ( offset > 0 ) { + minorVersion--; + if ( minorVersion < 0 ) { + majorVersion--; + minorVersion = 9; + } + offset--; + } + + // Set the version that we found in the offset. + wpVersion = majorVersion + '.' + minorVersion; + } + } + + // Scan through all of the versions to find the latest version in the version line. + let latestVersion = null; + let latestPatch = -1; + for ( const v in allVersions ) { + // Parse the version so we can make sure we're looking for the latest. + const matches = v.match( /([0-9]+)\.([0-9]+)(?:\.([0-9]+))?/ ); + + // We only care about the correct minor version. + const minor = `${ matches[ 1 ] }.${ matches[ 2 ] }`; + if ( minor !== wpVersion ) { + continue; + } + + // Track the latest version in the line. + const patch = + matches[ 3 ] === undefined ? 0 : parseInt( matches[ 3 ], 10 ); + + if ( patch > latestPatch ) { + latestPatch = patch; + latestVersion = v; + } + } + + if ( ! latestVersion ) { + throw new Error( + `Unable to find latest version for version line ${ wpVersion }.` + ); + } + + return `https://wordpress.org/wordpress-${ latestVersion }.zip`; +} + +/** + * Parses a display-friendly WordPress version and returns a link to download the given version. + * + * @param {string} wpVersion A display-friendly WordPress version. Supports ("master", "trunk", "nightly", "latest", "latest-X", "X.X" for version lines, and "X.X.X" for specific versions) + * @return {Promise.} A link to download the given version of WordPress. + */ +async function parseWPVersion( wpVersion: string ): Promise< string > { + // Allow for download URLs in place of a version. + if ( wpVersion.match( /[a-z]+:\/\//i ) ) { + return wpVersion; + } + + // Start with versions we can infer immediately. + switch ( wpVersion ) { + case 'master': + case 'trunk': { + return 'WordPress/WordPress#master'; + } + + case 'nightly': { + return 'https://wordpress.org/nightly-builds/wordpress-latest.zip'; + } + + case 'latest': { + return 'https://wordpress.org/latest.zip'; + } + } + + // We can also infer X.X.X versions immediately. + const parsedVersion = wpVersion.match( /^([0-9]+)\.([0-9]+)\.([0-9]+)$/ ); + if ( parsedVersion ) { + // Note that X.X.0 versions use a X.X download URL. + let urlVersion = `${ parsedVersion[ 1 ] }.${ parsedVersion[ 2 ] }`; + if ( parsedVersion[ 3 ] !== '0' ) { + urlVersion += `.${ parsedVersion[ 3 ] }`; + } + + return `https://wordpress.org/wordpress-${ urlVersion }.zip`; + } + + // Since we haven't found a URL yet we're going to use the WordPress.org API to try and infer one. + return getPreciseWPVersionURL( wpVersion ); +} + +/** + * The environment variables that should be set for the test environment. + */ +export interface TestEnvVars { + WP_ENV_CORE?: string; + WP_ENV_PHP_VERSION?: string; +} + +/** + * Parses the test environment's configuration and returns any environment variables that + * should be set. + * + * @param {Object} config The test environment configuration. + * @return {Promise.} The environment variables for the test environment. + */ +export async function parseTestEnvConfig( + config: TestEnvConfigVars +): Promise< TestEnvVars > { + const envVars: TestEnvVars = {}; + + // Convert `wp-env` configuration options to environment variables. + if ( config.wpVersion ) { + try { + envVars.WP_ENV_CORE = await parseWPVersion( config.wpVersion ); + } catch ( error ) { + throw new Error( + `Failed to parse WP version: ${ error.message }.` + ); + } + } + if ( config.phpVersion ) { + envVars.WP_ENV_PHP_VERSION = config.phpVersion; + } + + return envVars; +} diff --git a/tools/monorepo-utils/src/index.ts b/tools/monorepo-utils/src/index.ts index 11327b34925..8f49e79901c 100755 --- a/tools/monorepo-utils/src/index.ts +++ b/tools/monorepo-utils/src/index.ts @@ -13,6 +13,7 @@ import CodeFreeze from './code-freeze/commands'; import Slack from './slack/commands/slack'; import Manifest from './md-docs/commands'; import Changefile from './changefile'; +import CIJobs from './ci-jobs'; import WorkflowProfiler from './workflow-profiler/commands'; import { Logger } from './core/logger'; import { isGithubCI } from './core/environment'; @@ -33,6 +34,7 @@ const program = new Command() .addCommand( CodeFreeze ) .addCommand( Slack ) .addCommand( Changefile ) + .addCommand( CIJobs ) .addCommand( WorkflowProfiler ) .addCommand( Manifest );