Steve-Dee-Designs
/
woocommerce
mirror of https://github.com/woocommerce/woocommerce.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Include e2e, api and performance tests in ci.yml (#45190) 

Co-authored-by: Christopher Allford <6451942+ObliviousHarmony@users.noreply.github.com>
Co-authored-by: Ron Rennick <ron@ronandandrea.com>
This commit is contained in:
Adrian Moldovan 2024-03-27 17:21:49 +02:00 committed by GitHub
parent bb10ee5e57
commit 7d6d2c94dd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 858 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

280
.github/workflows/ci.yml vendored
View File

@ -1,5 +1,5 @@
name: 'CI'
on:
on:
pull_request:
push:
branches:
@ -8,6 +8,7 @@ on:
concurrency:
group: '${{ github.workflow }}-${{ github.ref }}'
cancel-in-progress: true
jobs:
project-jobs:
# Since this is a monorepo, not every pull request or change is going to impact every project.
@ -18,9 +19,12 @@ jobs:
runs-on: 'ubuntu-20.04'
outputs:
lint-jobs: ${{ steps.project-jobs.outputs.lint-jobs }}
test-jobs: ${{ steps.project-jobs.outputs.test-jobs }}
default-test-jobs: ${{ steps.project-jobs.outputs.default-test-jobs }}
e2e-test-jobs: ${{ steps.project-jobs.outputs.e2e-test-jobs }}
api-test-jobs: ${{ steps.project-jobs.outputs.api-test-jobs }}
performance-test-jobs: ${{ steps.project-jobs.outputs.performance-test-jobs }}
steps:
- uses: 'actions/checkout@v3'
- uses: 'actions/checkout@v4'
name: 'Checkout'
with:
fetch-depth: 0
@ -39,6 +43,7 @@ jobs:
}
const child_process = require( 'node:child_process' );
child_process.execSync( `pnpm utils ci-jobs ${ baseRef }` );
project-lint-jobs:
name: 'Lint - ${{ matrix.projectName }}'
runs-on: 'ubuntu-20.04'
@ -49,7 +54,7 @@ jobs:
matrix:
include: ${{ fromJSON( needs.project-jobs.outputs.lint-jobs ) }}
steps:
- uses: 'actions/checkout@v3'
- uses: 'actions/checkout@v4'
name: 'Checkout'
with:
fetch-depth: 0
@ -61,20 +66,19 @@ jobs:
build: '${{ matrix.projectName }}'
- name: 'Lint'
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
project-test-jobs:
project-default-test-jobs:
name: 'Test - ${{ matrix.projectName }} - ${{ matrix.name }}'
runs-on: 'ubuntu-20.04'
needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.test-jobs != '[]' }}
if: ${{ needs.project-jobs.outputs.default-test-jobs != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON( needs.project-jobs.outputs.test-jobs ) }}
include: ${{ fromJSON( needs.project-jobs.outputs.default-test-jobs ) }}
steps:
- uses: 'actions/checkout@v3'
- uses: 'actions/checkout@v4'
name: 'Checkout'
with:
fetch-depth: 0
- uses: './.github/actions/setup-woocommerce-monorepo'
name: 'Setup Monorepo'
id: 'setup-monorepo'
@ -88,18 +92,123 @@ jobs:
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
- name: 'Test'
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
project-e2e-test-jobs:
name: 'E2E - ${{ matrix.name }}'
runs-on: 'ubuntu-20.04'
needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.e2e-test-jobs != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON( needs.project-jobs.outputs.e2e-test-jobs ) }}
steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
- uses: './.github/actions/setup-woocommerce-monorepo'
name: 'Setup Monorepo'
id: 'setup-monorepo'
with:
install: '${{ matrix.projectName }}...'
build: '${{ matrix.projectName }}'
- name: 'Prepare Test Environment'
id: 'prepare-test-environment'
if: ${{ matrix.testEnv.shouldCreate }}
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
- name: 'Run tests'
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
- name: 'Upload artifacts'
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: all-blob-reports-${{ matrix.shardNumber }}
path: ${{ matrix.projectPath }}/tests/e2e-pw/test-results/allure-results
retention-days: 1
compression-level: 9
project-api-test-jobs:
name: 'API - ${{ matrix.name }}'
runs-on: 'ubuntu-20.04'
needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.api-test-jobs != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON( needs.project-jobs.outputs.api-test-jobs ) }}
steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
- uses: './.github/actions/setup-woocommerce-monorepo'
name: 'Setup Monorepo'
id: 'setup-monorepo'
with:
install: '${{ matrix.projectName }}...'
build: '${{ matrix.projectName }}'
- name: 'Prepare Test Environment'
id: 'prepare-test-environment'
if: ${{ matrix.testEnv.shouldCreate }}
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
- name: 'Run tests'
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
- name: 'Upload artifacts'
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: all-blob-reports-${{ matrix.shardNumber }}
path: ${{ matrix.projectPath }}/tests/api-core-tests/test-results/allure-results
retention-days: 1
compression-level: 9
project-performance-test-jobs:
name: 'Performance - ${{ matrix.name }}'
runs-on: 'ubuntu-20.04'
needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.performance-test-jobs != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON( needs.project-jobs.outputs.performance-test-jobs ) }}
steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
- uses: './.github/actions/setup-woocommerce-monorepo'
name: 'Setup Monorepo'
id: 'setup-monorepo'
with:
install: '${{ matrix.projectName }}...'
build: '${{ matrix.projectName }}'
- name: 'Prepare Test Environment'
id: 'prepare-test-environment'
if: ${{ matrix.testEnv.shouldCreate }}
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
- name: 'Run tests'
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
evaluate-project-jobs:
# In order to add a required status check we need a consistent job that we can grab onto.
# Since we are dynamically generating a matrix for the project jobs, however, we can't
# rely on on any specific job being present. We can get around this limitation by
# rely on any specific job being present. We can get around this limitation by
# using a job that runs after all the others and either passes or fails based
# on the results of the other jobs in the workflow.
name: 'Evaluate Project Job Statuses'
runs-on: 'ubuntu-20.04'
needs: [
needs: [
'project-jobs',
'project-lint-jobs',
'project-test-jobs'
'project-default-test-jobs',
'project-e2e-test-jobs',
'project-api-test-jobs'
]
if: ${{ always() }}
steps:
@ -115,9 +224,150 @@ jobs:
echo "One or more lint jobs have failed."
exit 1
fi
result="${{ needs.project-test-jobs.result }}"
result="${{ needs.project-default-test-jobs.result }}"
if [[ $result != "success" && $result != "skipped" ]]; then
echo "One or more test jobs have failed."
exit 1
fi
echo "All jobs have completed successfully."
result="${{ needs.project-e2e-test-jobs.result }}"
if [[ $result != "success" && $result != "skipped" ]]; then
echo "One or more e2e test jobs have failed."
exit 1
fi
result="${{ needs.project-api-test-jobs.result }}"
if [[ $result != "success" && $result != "skipped" ]]; then
echo "One or more api test jobs have failed."
exit 1
fi
echo "All jobs have completed successfully."
publish-e2e-test-reports:
name: 'Publish e2e test reports'
needs: [ project-e2e-test-jobs ]
if: ${{ contains( needs.*.result, 'success' ) || contains( needs.*.result, 'failure' ) }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: 'Install Allure CLI'
env:
DESTINATION_PATH: ../
run: ./.github/workflows/scripts/install-allure.sh
- name: 'Download blob reports from artifacts'
uses: actions/download-artifact@v4
with:
path: ./out/allure-results
pattern: all-blob-reports-*
run-id: project-e2e-test-jobs
merge-multiple: true
- name: 'Generate Allure report'
id: generate_allure_report
run: allure generate --clean ./out/allure-results --output ./out/allure-report
- name: 'Archive reports'
uses: actions/upload-artifact@v4
with:
name: e2e-test-report
path: ./out
if-no-files-found: ignore
retention-days: 5
- name: 'Publish reports'
env:
GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
RUN_ID: ${{ github.run_id }}
COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
run: |
if [ "$GITHUB_EVENT_NAME" == pull_request ]; then
gh workflow run publish-test-reports-pr.yml \
-f run_id=$RUN_ID \
-f e2e_artifact=e2e-test-report \
-f pr_number=$PR_NUMBER \
-f commit_sha=$COMMIT_SHA \
-f s3_root=public \
--repo woocommerce/woocommerce-test-reports
else
gh workflow run publish-test-reports-trunk-merge.yml \
-f run_id=$RUN_ID \
-f artifact=e2e-test-report \
-f pr_number='' \
-f test_type="e2e" \
--repo woocommerce/woocommerce-test-reports
fi
slack-alert-e2e-test:
name: 'Send Slack alert for e2e tests'
needs: [ project-e2e-test-jobs ]
if: ${{ github.event_name != 'pull_request' && ( contains( needs.*.result, 'success' ) || contains( needs.*.result, 'failure' ) ) }}
runs-on: ubuntu-latest
steps:
- name: 'Send Slack notification'
uses: automattic/action-test-results-to-slack@v0.3.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
slack_token: ${{ secrets.E2E_SLACK_TOKEN }}
slack_channel: ${{ secrets.E2E_TRUNK_SLACK_CHANNEL }}
publish-api-test-reports:
name: 'Publish API test reports'
needs: [ project-api-test-jobs ]
if: ${{ contains( needs.*.result, 'success' ) || contains( needs.*.result, 'failure' ) }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: 'Install Allure CLI'
env:
DESTINATION_PATH: ../
run: ./.github/workflows/scripts/install-allure.sh
- name: 'Download blob reports from artifacts'
uses: actions/download-artifact@v4
with:
path: ./out/allure-results
pattern: all-blob-reports-*
run-id: project-api-test-jobs
merge-multiple: true
- name: 'Generate Allure report'
id: generate_allure_report
run: allure generate --clean ./out/allure-results --output ./out/allure-report
- name: 'Archive reports'
uses: actions/upload-artifact@v4
with:
name: api-test-report
path: ./out
if-no-files-found: ignore
retention-days: 5
- name: 'Publish reports'
env:
GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
RUN_ID: ${{ github.run_id }}
COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
run: |
if [ "$GITHUB_EVENT_NAME" == pull_request ]; then
gh workflow run publish-test-reports-pr.yml \
-f run_id=$RUN_ID \
-f api_artifact=api-test-report \
-f pr_number=$PR_NUMBER \
-f commit_sha=$COMMIT_SHA \
-f s3_root=public \
--repo woocommerce/woocommerce-test-reports
else
gh workflow run publish-test-reports-trunk-merge.yml \
-f run_id=$RUN_ID \
-f artifact=api-test-report \
-f pr_number='' \
-f test_type="api" \
--repo woocommerce/woocommerce-test-reports
fi

6
.github/workflows/pr-build-and-e2e-tests.yml vendored
View File

@ -1,9 +1,9 @@
name: Run tests against PR
on:
workflow_dispatch:
pull_request:
paths-ignore:
- '**/changelog/**'
#pull_request:
#paths-ignore:
#- '**/changelog/**'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}

29
.github/workflows/scripts/install-allure.sh vendored Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
set -eo pipefail
# Test Java installation
java -version
if [[ -z "$DESTINATION_PATH" ]]; then
echo "::error::DESTINATION_PATH must be set"
exit 1
fi
ALLURE_VERSION=2.27.0
ALLURE_DOWNLOAD_URL=https://github.com/allure-framework/allure2/releases/download/$ALLURE_VERSION/allure-$ALLURE_VERSION.zip
echo "Installing Allure $ALLURE_VERSION in $DESTINATION_PATH"
wget --no-verbose -O allure.zip $ALLURE_DOWNLOAD_URL \
&& unzip allure.zip -d "$DESTINATION_PATH" \
&& rm -rf allure.zip \
ALLURE_PATH=$(realpath "$DESTINATION_PATH"/allure-$ALLURE_VERSION/bin)
# Test Allure installation
echo "$ALLURE_PATH"
export PATH="$ALLURE_PATH:$PATH"
allure --version
# Add Allure in Github PATH to make it available to all subsequent actions in the current job
echo "$ALLURE_PATH" >> "$GITHUB_PATH"

7
.github/workflows/smoke-test-pr-merge.yml vendored
View File

@ -1,8 +1,9 @@
name: Run tests against trunk after PR merge
on:
pull_request:
types:
- closed
workflow_dispatch:
#pull_request:
#types:
#- closed
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true

4
plugins/woocommerce/changelog/tools-add-e2e-tests-in-ci-jobs Normal file
View File

@ -0,0 +1,4 @@
Significance: minor
Type: dev
CI: adds e2e tests into ci-jobs and ci.yml

39
plugins/woocommerce/package.json
View File

@ -28,8 +28,10 @@
"env:restart": "pnpm wp-env destroy && pnpm wp-env start --update",
"env:start": "pnpm wp-env start",
"env:stop": "pnpm wp-env stop",
"env:test": "WP_ENV_LIFECYCLE_SCRIPT_AFTER_START='./tests/e2e-pw/bin/test-env-setup.sh' && pnpm env:dev",
"env:test": "WP_ENV_LIFECYCLE_SCRIPT_AFTER_START='./tests/e2e-pw/bin/test-env-setup.sh' && pnpm env:dev && pnpm playwright install chromium",
"env:test:cot": "WP_ENV_LIFECYCLE_SCRIPT_AFTER_START='ENABLE_HPOS=1 ./tests/e2e-pw/bin/test-env-setup.sh' && ENABLE_HPOS=1 pnpm env:dev",
"env:perf:install-k6": "curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1",
"env:perf": "pnpm env:dev && pnpm env:performance-init && pnpm env:perf:install-k6",
"preinstall": "npx only-allow pnpm",
"postinstall": "composer install",
"lint": "pnpm --if-present '/^lint:lang:.*$/'",
@ -48,6 +50,7 @@
"test:api": "API_TEST_REPORT_DIR=\"$PWD/tests/api\" pnpm exec wc-api-tests test api",
"test:api-pw": "USE_WP_ENV=1 pnpm playwright test --config=tests/api-core-tests/playwright.config.js",
"test:e2e-pw": "USE_WP_ENV=1 pnpm playwright test --config=tests/e2e-pw/playwright.config.js",
"test:perf": "./k6 run ./tests/performance/tests/gh-action-pr-requests.js",
"test:env:start": "pnpm env:test",
"test:php": "./vendor/bin/phpunit -c ./phpunit.xml",
"test:php:watch": "./vendor/bin/phpunit-watcher watch",
@ -165,6 +168,40 @@
"wpVersion": "latest-2"
}
}
},
{
"name": "Core e2e tests",
"testType": "e2e",
"command": "test:e2e-pw",
"shardingArguments": [
"--shard=1/5",
"--shard=2/5",
"--shard=3/5",
"--shard=4/5",
"--shard=5/5"
],
"changes": [],
"testEnv": {
"start": "env:test"
}
},
{
"name": "Core API tests",
"testType": "api",
"command": "test:api-pw",
"changes": [],
"testEnv": {
"start": "env:test"
}
},
{
"name": "Core Performance tests (K6)",
"testType": "performance",
"command": "test:perf",
"changes": [],
"testEnv": {
"start": "env:perf"
}
}
]
}

2
tools/monorepo-utils/dist/index.js vendored
View File

File diff suppressed because one or more lines are too long

25
tools/monorepo-utils/src/ci-jobs/index.ts
View File

@ -12,6 +12,7 @@ import { buildProjectGraph } from './lib/project-graph';
import { getFileChanges } from './lib/file-changes';
import { createJobsForChanges } from './lib/job-processing';
import { isGithubCI } from '../core/environment';
import { testTypes } from './lib/config';
const program = new Command( 'ci-jobs' )
.description(
@ -49,7 +50,13 @@ const program = new Command( 'ci-jobs' )
if ( isGithubCI() ) {
setOutput( 'lint-jobs', JSON.stringify( jobs.lint ) );
setOutput( 'test-jobs', JSON.stringify( jobs.test ) );
testTypes.forEach( ( type ) => {
setOutput(
`${ type }-test-jobs`,
JSON.stringify( jobs[ `${ type }Test` ] )
);
} );
return;
}
@ -62,14 +69,16 @@ const program = new Command( 'ci-jobs' )
Logger.notice( 'No lint jobs to run.' );
}
if ( jobs.test.length > 0 ) {
Logger.notice( 'Test Jobs' );
for ( const job of jobs.test ) {
Logger.notice( `- ${ job.projectName } - ${ job.name }` );
testTypes.forEach( ( type ) => {
if ( jobs[ `${ type }Test` ].length > 0 ) {
Logger.notice( `${ type } test Jobs` );
for ( const job of jobs[ `${ type }Test` ] ) {
Logger.notice( `- ${ job.projectName } - ${ job.name }` );
}
} else {
Logger.notice( `No ${ type } test jobs to run.` );
}
} else {
Logger.notice( 'No test jobs to run.' );
}
} );
} );
export default program;

128
tools/monorepo-utils/src/ci-jobs/lib/__tests__/config.spec.ts
View File

@ -6,7 +6,7 @@ import { makeRe } from 'minimatch';
/**
* Internal dependencies
*/
import { JobType, parseCIConfig } from '../config';
import { JobType, parseCIConfig, testTypes } from '../config';
describe( 'Config', () => {
describe( 'parseCIConfig', () => {
@ -136,6 +136,8 @@ describe( 'Config', () => {
jobs: [
{
type: JobType.Test,
testType: 'default',
shardingArguments: [],
name: 'default',
changes: [
/^package\.json$/,
@ -222,5 +224,129 @@ describe( 'Config', () => {
],
} );
} );
it.each( testTypes )(
'should parse test config with expected testType',
( testType ) => {
const parsed = parseCIConfig( {
name: 'foo',
config: {
ci: {
tests: [
{
name: 'default',
testType,
changes: '/src/**/*.{js,jsx,ts,tsx}',
command: 'foo',
},
],
},
},
} );
expect( parsed ).toMatchObject( {
jobs: [
{
type: JobType.Test,
testType,
shardingArguments: [],
name: 'default',
changes: [
/^package\.json$/,
makeRe( '/src/**/*.{js,jsx,ts,tsx}' ),
],
command: 'foo',
},
],
} );
}
);
it.each( [
[ '', 'default' ],
[ 'bad', 'default' ],
[ 1, 'default' ],
[ undefined, 'default' ],
] )(
'should parse test config with unexpected testType',
( input, result ) => {
const parsed = parseCIConfig( {
name: 'foo',
config: {
ci: {
tests: [
{
name: 'default',
testType: input,
changes: '/src/**/*.{js,jsx,ts,tsx}',
command: 'foo',
},
],
},
},
} );
expect( parsed ).toMatchObject( {
jobs: [
{
type: JobType.Test,
testType: result,
shardingArguments: [],
name: 'default',
changes: [
/^package\.json$/,
makeRe( '/src/**/*.{js,jsx,ts,tsx}' ),
],
command: 'foo',
},
],
} );
}
);
it.each( [
[ [], [] ],
[ undefined, [] ],
[
[ 'a', 'b' ],
[ 'a', 'b' ],
],
] )(
'should parse test config with shards',
( shardingArguments: any, result: string[] ) => {
const parsed = parseCIConfig( {
name: 'foo',
config: {
ci: {
tests: [
{
name: 'default',
testType: 'e2e',
shardingArguments,
changes: '/src/**/*.{js,jsx,ts,tsx}',
command: 'foo',
},
],
},
},
} );
expect( parsed ).toMatchObject( {
jobs: [
{
type: JobType.Test,
testType: 'e2e',
shardingArguments: result,
name: 'default',
changes: [
/^package\.json$/,
makeRe( '/src/**/*.{js,jsx,ts,tsx}' ),
],
command: 'foo',
},
],
} );
}
);
} );
} );

303
tools/monorepo-utils/src/ci-jobs/lib/__tests__/job-processing.spec.ts
View File

@ -1,8 +1,8 @@
/**
* Internal dependencies
*/
import { JobType } from '../config';
import { createJobsForChanges } from '../job-processing';
import { JobType, testTypes } from '../config';
import { createJobsForChanges, getShardedJobs } from '../job-processing';
import { parseTestEnvConfig } from '../test-environment';
jest.mock( '../test-environment' );
@ -49,6 +49,7 @@ describe( 'Job Processing', () => {
expect( jobs.lint ).toHaveLength( 1 );
expect( jobs.lint ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
command: 'test-lint',
} );
expect( jobs.test ).toHaveLength( 0 );
@ -83,6 +84,7 @@ describe( 'Job Processing', () => {
expect( jobs.lint ).toHaveLength( 1 );
expect( jobs.lint ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
command: 'test-lint test-base-ref',
} );
expect( jobs.test ).toHaveLength( 0 );
@ -220,10 +222,12 @@ describe( 'Job Processing', () => {
expect( jobs.lint ).toHaveLength( 2 );
expect( jobs.lint ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
command: 'test-lint',
} );
expect( jobs.lint ).toContainEqual( {
projectName: 'test-b',
projectPath: 'test-b',
command: 'test-lint-b',
} );
expect( jobs.test ).toHaveLength( 0 );
@ -276,16 +280,20 @@ describe( 'Job Processing', () => {
expect( jobs.lint ).toHaveLength( 2 );
expect( jobs.lint ).toContainEqual( {
projectName: 'test-a',
projectPath: 'test-a',
command: 'test-lint-a',
} );
expect( jobs.lint ).toContainEqual( {
projectName: 'test-b',
projectPath: 'test-b',
command: 'test-lint-b',
} );
expect( jobs.test ).toHaveLength( 0 );
} );
it( 'should trigger test job for single node', async () => {
const testType = 'default';
const jobs = await createJobsForChanges(
{
name: 'test',
@ -294,6 +302,8 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType,
shardingArguments: [],
name: 'Default',
changes: [ /test.js$/ ],
command: 'test-cmd',
@ -309,11 +319,13 @@ describe( 'Job Processing', () => {
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs.test ).toHaveLength( 1 );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
@ -322,6 +334,7 @@ describe( 'Job Processing', () => {
} );
it( 'should replace vars in test command', async () => {
const testType = 'default';
const jobs = await createJobsForChanges(
{
name: 'test',
@ -330,7 +343,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [],
changes: [ /test.js$/ ],
command: 'test-cmd <baseRef>',
},
@ -349,11 +364,13 @@ describe( 'Job Processing', () => {
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs.test ).toHaveLength( 1 );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd test-base-ref',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
@ -362,6 +379,7 @@ describe( 'Job Processing', () => {
} );
it( 'should not trigger a test job that has already been created', async () => {
const testType = 'default';
const jobs = await createJobsForChanges(
{
name: 'test',
@ -370,7 +388,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [],
changes: [ /test.js$/ ],
command: 'test-cmd',
jobCreated: true,
@ -386,10 +406,11 @@ describe( 'Job Processing', () => {
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs.test ).toHaveLength( 0 );
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 0 );
} );
it( 'should not trigger test job for single node with no changes', async () => {
const testType = 'default';
const jobs = await createJobsForChanges(
{
name: 'test',
@ -398,7 +419,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [],
changes: [ /test.js$/ ],
command: 'test-cmd',
},
@ -411,10 +434,11 @@ describe( 'Job Processing', () => {
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs.test ).toHaveLength( 0 );
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 0 );
} );
it( 'should trigger test job for project graph', async () => {
const testType = 'default';
const jobs = await createJobsForChanges(
{
name: 'test',
@ -423,7 +447,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [],
changes: [ /test.js$/ ],
command: 'test-cmd',
},
@ -437,7 +463,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType: 'default',
name: 'Default A',
shardingArguments: [],
changes: [ /test-b.js$/ ],
command: 'test-cmd-a',
},
@ -452,7 +480,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType: 'default',
name: 'Default B',
shardingArguments: [],
changes: [ /test-b.js$/ ],
command: 'test-cmd-b',
},
@ -471,20 +501,24 @@ describe( 'Job Processing', () => {
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs.test ).toHaveLength( 2 );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 2 );
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
},
} );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test-b',
projectPath: 'test-b',
name: 'Default B',
command: 'test-cmd-b',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
@ -492,7 +526,51 @@ describe( 'Job Processing', () => {
} );
} );
it.each( testTypes )(
'should trigger %s test job for single node',
async ( testType ) => {
const jobs = await createJobsForChanges(
{
name: 'test',
path: 'test',
ciConfig: {
jobs: [
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [],
changes: [ /test.js$/ ],
command: 'test-cmd',
},
],
},
dependencies: [],
},
{
test: [ 'test.js' ],
},
{}
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
},
} );
}
);
it( 'should trigger test job for dependent without changes when dependency has matching cascade key', async () => {
const testType = 'default';
const jobs = await createJobsForChanges(
{
name: 'test',
@ -501,7 +579,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [],
changes: [ /test.js$/ ],
command: 'test-cmd',
cascadeKeys: [ 'test' ],
@ -516,7 +596,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType: 'default',
name: 'Default A',
shardingArguments: [],
changes: [ /test-a.js$/ ],
command: 'test-cmd-a',
cascadeKeys: [ 'test-a', 'test' ],
@ -534,20 +616,24 @@ describe( 'Job Processing', () => {
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs.test ).toHaveLength( 2 );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 2 );
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
},
} );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test-a',
projectPath: 'test-a',
name: 'Default A',
command: 'test-cmd-a',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
@ -556,6 +642,7 @@ describe( 'Job Processing', () => {
} );
it( 'should isolate dependency cascade keys to prevent cross-dependency matching', async () => {
const testType = 'default';
const jobs = await createJobsForChanges(
{
name: 'test',
@ -564,7 +651,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [],
changes: [ /test.js$/ ],
command: 'test-cmd',
cascadeKeys: [ 'test' ],
@ -579,7 +668,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType: 'default',
name: 'Default A',
shardingArguments: [],
changes: [ /test-a.js$/ ],
command: 'test-cmd-a',
cascadeKeys: [ 'test-a', 'test' ],
@ -595,7 +686,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType: 'default',
name: 'Default B',
shardingArguments: [],
changes: [ /test-b.js$/ ],
command: 'test-cmd-b',
cascadeKeys: [ 'test-b', 'test' ],
@ -613,20 +706,24 @@ describe( 'Job Processing', () => {
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs.test ).toHaveLength( 2 );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 2 );
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
},
} );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test-a',
projectPath: 'test-a',
name: 'Default A',
command: 'test-cmd-a',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
@ -635,6 +732,7 @@ describe( 'Job Processing', () => {
} );
it( 'should trigger test job for single node and parse test environment config', async () => {
const testType = 'default';
jest.mocked( parseTestEnvConfig ).mockResolvedValue( {
WP_ENV_CORE: 'https://wordpress.org/latest.zip',
} );
@ -647,7 +745,9 @@ describe( 'Job Processing', () => {
jobs: [
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [],
changes: [ /test.js$/ ],
command: 'test-cmd',
testEnv: {
@ -672,11 +772,13 @@ describe( 'Job Processing', () => {
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs.test ).toHaveLength( 1 );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: true,
start: 'test-start test-base-ref',
@ -688,6 +790,7 @@ describe( 'Job Processing', () => {
} );
it( 'should trigger all jobs for a single node with changes set to "true"', async () => {
const testType = 'default';
const jobs = await createJobsForChanges(
{
name: 'test',
@ -701,7 +804,9 @@ describe( 'Job Processing', () => {
},
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [],
changes: [ /test.js$/ ],
command: 'test-cmd',
},
@ -716,18 +821,174 @@ describe( 'Job Processing', () => {
expect( jobs.lint ).toHaveLength( 1 );
expect( jobs.lint ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
command: 'test-lint',
} );
expect( jobs.test ).toHaveLength( 1 );
expect( jobs.test ).toContainEqual( {
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
},
} );
} );
it( 'should trigger sharded test jobs for single node', async () => {
const testType = 'default';
const jobs = await createJobsForChanges(
{
name: 'test',
path: 'test',
ciConfig: {
jobs: [
{
type: JobType.Test,
testType,
name: 'Default',
shardingArguments: [
'--shard=1/2',
'--shard=2/2',
],
changes: [ /test.js$/ ],
command: 'test-cmd',
},
],
},
dependencies: [],
},
{
test: [ 'test.js' ],
},
{}
);
expect( jobs.lint ).toHaveLength( 0 );
expect( jobs[ `${ testType }Test` ] ).toHaveLength( 2 );
expect( jobs[ `${ testType }Test` ] ).toEqual(
expect.arrayContaining( [
{
projectName: 'test',
projectPath: 'test',
name: 'Default 1/2',
command: 'test-cmd --shard=1/2',
shardNumber: 1,
testEnv: {
shouldCreate: false,
envVars: {},
},
},
{
projectName: 'test',
projectPath: 'test',
name: 'Default 2/2',
command: 'test-cmd --shard=2/2',
shardNumber: 2,
testEnv: {
shouldCreate: false,
envVars: {},
},
},
] )
);
} );
} );
describe( 'getShardedJobs', () => {
it( 'should create sharded jobs', async () => {
const jobs = getShardedJobs(
{
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
},
},
{
type: JobType.Test,
testType: 'e2e',
name: 'Default',
shardingArguments: [ '--shard-arg-1', '--shard-arg-2' ],
changes: [ /test.js$/ ],
command: 'test-cmd',
}
);
expect( jobs ).toHaveLength( 2 );
expect( jobs ).toEqual(
expect.arrayContaining( [
{
projectName: 'test',
projectPath: 'test',
name: 'Default 1/2',
command: 'test-cmd --shard-arg-1',
shardNumber: 1,
testEnv: {
shouldCreate: false,
envVars: {},
},
},
{
projectName: 'test',
projectPath: 'test',
name: 'Default 2/2',
command: 'test-cmd --shard-arg-2',
shardNumber: 2,
testEnv: {
shouldCreate: false,
envVars: {},
},
},
] )
);
} );
it.each( [ [ [] ], [ [ '--sharding=1/1' ] ] ] )(
'should not create sharded jobs for shards',
async ( shardingArguments ) => {
const jobs = getShardedJobs(
{
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
},
},
{
type: JobType.Test,
testType: 'e2e',
name: 'Default',
shardingArguments,
changes: [ /test.js$/ ],
command: 'test-cmd',
}
);
expect( jobs ).toHaveLength( 1 );
expect( jobs ).toContainEqual( {
projectName: 'test',
projectPath: 'test',
name: 'Default',
command: 'test-cmd',
shardNumber: 0,
testEnv: {
shouldCreate: false,
envVars: {},
},
} );
}
);
} );
} );

25
tools/monorepo-utils/src/ci-jobs/lib/config.ts
View File

@ -21,6 +21,11 @@ export const enum JobType {
Test = 'test',
}
/**
* The type of the test job.
*/
export const testTypes = [ 'default', 'e2e', 'api', 'performance' ] as const;
/**
* The variables that can be used in tokens on command strings
* that will be replaced during job creation.
@ -251,11 +256,21 @@ export interface TestJobConfig extends BaseJobConfig {
*/
type: JobType.Test;
/**
* The type of the test.
*/
testType: ( typeof testTypes )[ number ];
/**
* The name for the job.
*/
name: string;
/**
* The number of shards to be created for this job.
*/
shardingArguments: string[];
/**
* The configuration for the test environment if one is needed.
*/
@ -320,10 +335,20 @@ function parseTestJobConfig( raw: any ): TestJobConfig {
);
}
let testType: ( typeof testTypes )[ number ] = 'default';
if (
raw.testType &&
testTypes.includes( raw.testType.toString().toLowerCase() )
) {
testType = raw.testType.toLowerCase();
}
validateCommandVars( raw.command );
const config: TestJobConfig = {
type: JobType.Test,
testType,
shardingArguments: raw.shardingArguments || [],
name: raw.name,
changes: parseChangesConfig( raw.changes, [ 'package.json' ] ),
command: raw.command,

67
tools/monorepo-utils/src/ci-jobs/lib/job-processing.ts
View File

@ -4,6 +4,7 @@
import {
CommandVarOptions,
JobType,
testTypes,
LintJobConfig,
TestJobConfig,
} from './config';
@ -16,6 +17,7 @@ import { TestEnvVars, parseTestEnvConfig } from './test-environment';
*/
interface LintJob {
projectName: string;
projectPath: string;
command: string;
}
@ -33,9 +35,11 @@ interface TestJobEnv {
*/
interface TestJob {
projectName: string;
projectPath: string;
name: string;
command: string;
testEnv: TestJobEnv;
shardNumber: number;
}
/**
@ -70,10 +74,42 @@ function replaceCommandVars( command: string, options: CreateOptions ): string {
} );
}
/**
* Multiplies a job based on the shards job config. It updates the job names and command - currently only supporting Playwright sharding.
*
* @param {TestJob} job The job to be multiplied.
* @param {TestJobConfig} jobConfig The job config.
* @return {TestJob[]} The list of sharded jobs.
*/
export function getShardedJobs(
job: TestJob,
jobConfig: TestJobConfig
): TestJob[] {
let createdJobs = [];
const shards = jobConfig.shardingArguments.length;
if ( shards <= 1 ) {
createdJobs.push( job );
} else {
createdJobs = Array( shards )
.fill( null )
.map( ( _, i ) => {
const jobCopy = JSON.parse( JSON.stringify( job ) );
jobCopy.shardNumber = i + 1;
jobCopy.name = `${ job.name } ${ i + 1 }/${ shards }`;
jobCopy.command = `${ job.command } ${ jobConfig.shardingArguments[ i ] }`;
return jobCopy;
} );
}
return createdJobs;
}
/**
* Checks the config against the changes and creates one if it should be run.
*
* @param {string} projectName The name of the project that the job is for.
* @param {string} projectPath The path of the project that the job is for.
* @param {Object} config The config object for the lint job.
* @param {Array.<string>|true} changes The file changes that have occurred for the project or true if all projects should be marked as changed.
* @param {Object} options The options to use when creating the job.
@ -81,6 +117,7 @@ function replaceCommandVars( command: string, options: CreateOptions ): string {
*/
function createLintJob(
projectName: string,
projectPath: string,
config: LintJobConfig,
changes: string[] | true,
options: CreateOptions
@ -114,6 +151,7 @@ function createLintJob(
return {
projectName,
projectPath,
command: replaceCommandVars( config.command, options ),
};
}
@ -122,18 +160,22 @@ function createLintJob(
* Checks the config against the changes and creates one if it should be run.
*
* @param {string} projectName The name of the project that the job is for.
* @param {string} projectPath The path of the project that the job is for.
* @param {Object} config The config object for the test job.
* @param {Array.<string>|true} changes The file changes that have occurred for the project or true if all projects should be marked as changed.
* @param {Object} options The options to use when creating the job.
* @param {Array.<string>} cascadeKeys The cascade keys that have been triggered in dependencies.
* @param {number} shardNumber The shard number for the job.
* @return {Promise.<Object|null>} The job that should be run or null if no job should be run.
*/
async function createTestJob(
projectName: string,
projectPath: string,
config: TestJobConfig,
changes: string[] | true,
options: CreateOptions,
cascadeKeys: string[]
cascadeKeys: string[],
shardNumber: number
): Promise< TestJob | null > {
let triggered = false;
@ -179,12 +221,14 @@ async function createTestJob(
const createdJob: TestJob = {
projectName,
projectPath,
name: config.name,
command: replaceCommandVars( config.command, options ),
testEnv: {
shouldCreate: false,
envVars: {},
},
shardNumber,
};
// We want to make sure that we're including the configuration for
@ -221,6 +265,10 @@ async function createJobsForProject(
test: [],
};
testTypes.forEach( ( type ) => {
newJobs[ `${ type }Test` ] = [];
} );
// In order to simplify the way that cascades work we're going to recurse depth-first and check our dependencies
// for jobs before ourselves. This lets any cascade keys created in dependencies cascade to dependents.
const newCascadeKeys = [];
@ -240,7 +288,12 @@ async function createJobsForProject(
dependencyCascade
);
newJobs.lint.push( ...dependencyJobs.lint );
newJobs.test.push( ...dependencyJobs.test );
testTypes.forEach( ( type ) => {
newJobs[ `${ type }Test` ].push(
...dependencyJobs[ `${ type }Test` ]
);
} );
// Track any new cascade keys added by the dependency.
// Since we're filtering out duplicates after the
@ -285,6 +338,7 @@ async function createJobsForProject(
case JobType.Lint: {
const created = createLintJob(
node.name,
node.path,
jobConfig,
projectChanges,
options
@ -301,17 +355,22 @@ async function createJobsForProject(
case JobType.Test: {
const created = await createTestJob(
node.name,
node.path,
jobConfig,
projectChanges,
options,
cascadeKeys
cascadeKeys,
0
);
if ( ! created ) {
break;
}
jobConfig.jobCreated = true;
newJobs.test.push( created );
newJobs[ `${ jobConfig.testType }Test` ].push(
...getShardedJobs( created, jobConfig )
);
// We need to track any cascade keys that this job is associated with so that
// dependent projects can trigger jobs with matching keys. We are expecting