Include e2e, api and performance tests in ci.yml (#45190)

Co-authored-by: Christopher Allford <6451942+ObliviousHarmony@users.noreply.github.com>
Co-authored-by: Ron Rennick <ron@ronandandrea.com>

.github/workflows/ci.yml

@@ -8,6 +8,7 @@ on:
 concurrency:
   group: '${{ github.workflow }}-${{ github.ref }}'
   cancel-in-progress: true
+
 jobs:
   project-jobs:
     # Since this is a monorepo, not every pull request or change is going to impact every project.
@@ -18,9 +19,12 @@ jobs:
     runs-on: 'ubuntu-20.04'
     outputs:
       lint-jobs: ${{ steps.project-jobs.outputs.lint-jobs }}
-      test-jobs: ${{ steps.project-jobs.outputs.test-jobs }}
+      default-test-jobs: ${{ steps.project-jobs.outputs.default-test-jobs }}
+      e2e-test-jobs: ${{ steps.project-jobs.outputs.e2e-test-jobs }}
+      api-test-jobs: ${{ steps.project-jobs.outputs.api-test-jobs }}
+      performance-test-jobs: ${{ steps.project-jobs.outputs.performance-test-jobs }}
     steps:
-      - uses: 'actions/checkout@v3'
+      - uses: 'actions/checkout@v4'
         name: 'Checkout'
         with:
           fetch-depth: 0
@@ -39,6 +43,7 @@ jobs:
             }
             const child_process = require( 'node:child_process' );
             child_process.execSync( `pnpm utils ci-jobs ${ baseRef }` );
+
   project-lint-jobs:
     name: 'Lint - ${{ matrix.projectName }}'
     runs-on: 'ubuntu-20.04'
@@ -49,7 +54,7 @@ jobs:
       matrix:
         include: ${{ fromJSON( needs.project-jobs.outputs.lint-jobs ) }}
     steps:
-      - uses: 'actions/checkout@v3'
+      - uses: 'actions/checkout@v4'
         name: 'Checkout'
         with:
           fetch-depth: 0
@@ -61,20 +66,19 @@ jobs:
           build: '${{ matrix.projectName }}'
       - name: 'Lint'
         run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
-  project-test-jobs:
+
+  project-default-test-jobs:
     name: 'Test - ${{ matrix.projectName }} - ${{ matrix.name }}'
     runs-on: 'ubuntu-20.04'
     needs: 'project-jobs'
-    if: ${{ needs.project-jobs.outputs.test-jobs != '[]' }}
+    if: ${{ needs.project-jobs.outputs.default-test-jobs != '[]' }}
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJSON( needs.project-jobs.outputs.test-jobs ) }}
+        include: ${{ fromJSON( needs.project-jobs.outputs.default-test-jobs ) }}
     steps:
-      - uses: 'actions/checkout@v3'
+      - uses: 'actions/checkout@v4'
         name: 'Checkout'
-        with:
-          fetch-depth: 0
       - uses: './.github/actions/setup-woocommerce-monorepo'
         name: 'Setup Monorepo'
         id: 'setup-monorepo'
@@ -88,10 +92,113 @@ jobs:
         run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
       - name: 'Test'
         run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
+  
+  project-e2e-test-jobs:
+    name: 'E2E - ${{ matrix.name }}'
+    runs-on: 'ubuntu-20.04'
+    needs: 'project-jobs'
+    if: ${{ needs.project-jobs.outputs.e2e-test-jobs != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON( needs.project-jobs.outputs.e2e-test-jobs ) }}
+    steps:
+      - uses: 'actions/checkout@v4'
+        name: 'Checkout'
+
+      - uses: './.github/actions/setup-woocommerce-monorepo'
+        name: 'Setup Monorepo'
+        id: 'setup-monorepo'
+        with:
+          install: '${{ matrix.projectName }}...'
+          build: '${{ matrix.projectName }}'
+
+      - name: 'Prepare Test Environment'
+        id: 'prepare-test-environment'
+        if: ${{ matrix.testEnv.shouldCreate }}
+        run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
+
+      - name: 'Run tests'
+        run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
+
+      - name: 'Upload artifacts'
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: all-blob-reports-${{ matrix.shardNumber }}
+          path: ${{ matrix.projectPath }}/tests/e2e-pw/test-results/allure-results
+          retention-days: 1
+          compression-level: 9
+          
+  project-api-test-jobs:
+    name: 'API - ${{ matrix.name }}'
+    runs-on: 'ubuntu-20.04'
+    needs: 'project-jobs'
+    if: ${{ needs.project-jobs.outputs.api-test-jobs != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON( needs.project-jobs.outputs.api-test-jobs ) }}
+    steps:
+      - uses: 'actions/checkout@v4'
+        name: 'Checkout'
+
+      - uses: './.github/actions/setup-woocommerce-monorepo'
+        name: 'Setup Monorepo'
+        id: 'setup-monorepo'
+        with:
+          install: '${{ matrix.projectName }}...'
+          build: '${{ matrix.projectName }}'
+
+      - name: 'Prepare Test Environment'
+        id: 'prepare-test-environment'
+        if: ${{ matrix.testEnv.shouldCreate }}
+        run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
+
+      - name: 'Run tests'
+        run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
+
+      - name: 'Upload artifacts'
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: all-blob-reports-${{ matrix.shardNumber }}
+          path: ${{ matrix.projectPath }}/tests/api-core-tests/test-results/allure-results
+          retention-days: 1
+          compression-level: 9
+          
+  project-performance-test-jobs:
+    name: 'Performance - ${{ matrix.name }}'
+    runs-on: 'ubuntu-20.04'
+    needs: 'project-jobs'
+    if: ${{ needs.project-jobs.outputs.performance-test-jobs != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON( needs.project-jobs.outputs.performance-test-jobs ) }}
+    steps:
+      - uses: 'actions/checkout@v4'
+        name: 'Checkout'
+
+      - uses: './.github/actions/setup-woocommerce-monorepo'
+        name: 'Setup Monorepo'
+        id: 'setup-monorepo'
+        with:
+          install: '${{ matrix.projectName }}...'
+          build: '${{ matrix.projectName }}'
+
+      - name: 'Prepare Test Environment'
+        id: 'prepare-test-environment'
+        if: ${{ matrix.testEnv.shouldCreate }}
+        run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
+
+      - name: 'Run tests'
+        run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
+  
   evaluate-project-jobs:
     # In order to add a required status check we need a consistent job that we can grab onto.
     # Since we are dynamically generating a matrix for the project jobs, however, we can't
-    # rely on on any specific job being present. We can get around this limitation by
+    # rely on any specific job being present. We can get around this limitation by
     # using a job that runs after all the others and either passes or fails based
     # on the results of the other jobs in the workflow.
     name: 'Evaluate Project Job Statuses'
@@ -99,7 +206,9 @@ jobs:
     needs: [
       'project-jobs',
       'project-lint-jobs',
-      'project-test-jobs'
+      'project-default-test-jobs',
+      'project-e2e-test-jobs',
+      'project-api-test-jobs'
     ]
     if: ${{ always() }}
     steps:
@@ -115,9 +224,150 @@ jobs:
             echo "One or more lint jobs have failed."
             exit 1
           fi
-          result="${{ needs.project-test-jobs.result }}"
+          result="${{ needs.project-default-test-jobs.result }}"
           if [[ $result != "success" && $result != "skipped" ]]; then
             echo "One or more test jobs have failed."
             exit 1
           fi
+          result="${{ needs.project-e2e-test-jobs.result }}"
+          if [[ $result != "success" && $result != "skipped" ]]; then
+            echo "One or more e2e test jobs have failed."
+            exit 1
+          fi
+          result="${{ needs.project-api-test-jobs.result }}"
+          if [[ $result != "success" && $result != "skipped" ]]; then
+            echo "One or more api test jobs have failed."
+            exit 1
+          fi
           echo "All jobs have completed successfully."
+
+  publish-e2e-test-reports:
+    name: 'Publish e2e test reports'
+    needs: [ project-e2e-test-jobs ]
+    if: ${{ contains( needs.*.result, 'success' ) || contains( needs.*.result, 'failure' ) }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 'Install Allure CLI'
+        env:
+          DESTINATION_PATH: ../
+        run: ./.github/workflows/scripts/install-allure.sh
+
+      - name: 'Download blob reports from artifacts'
+        uses: actions/download-artifact@v4
+        with:
+          path: ./out/allure-results
+          pattern: all-blob-reports-*
+          run-id: project-e2e-test-jobs
+          merge-multiple: true
+
+      - name: 'Generate Allure report'
+        id: generate_allure_report
+        run: allure generate --clean ./out/allure-results --output ./out/allure-report
+
+      - name: 'Archive reports'
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-test-report
+          path: ./out
+          if-no-files-found: ignore
+          retention-days: 5
+          
+      - name: 'Publish reports'
+        env:
+          GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          RUN_ID: ${{ github.run_id }}
+          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          if [ "$GITHUB_EVENT_NAME" == pull_request ]; then
+            gh workflow run publish-test-reports-pr.yml \
+              -f run_id=$RUN_ID \
+              -f e2e_artifact=e2e-test-report \
+              -f pr_number=$PR_NUMBER \
+              -f commit_sha=$COMMIT_SHA \
+              -f s3_root=public \
+              --repo woocommerce/woocommerce-test-reports
+          else
+            gh workflow run publish-test-reports-trunk-merge.yml \
+              -f run_id=$RUN_ID \
+              -f artifact=e2e-test-report \
+              -f pr_number='' \
+              -f test_type="e2e" \
+              --repo woocommerce/woocommerce-test-reports
+          fi
+
+
+  slack-alert-e2e-test:
+    name: 'Send Slack alert for e2e tests'
+    needs: [ project-e2e-test-jobs ]
+    if: ${{ github.event_name != 'pull_request' && ( contains( needs.*.result, 'success' ) || contains( needs.*.result, 'failure' ) ) }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: 'Send Slack notification'
+        uses: automattic/action-test-results-to-slack@v0.3.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          slack_token: ${{ secrets.E2E_SLACK_TOKEN }}
+          slack_channel: ${{ secrets.E2E_TRUNK_SLACK_CHANNEL }}
+
+  publish-api-test-reports:
+    name: 'Publish API test reports'
+    needs: [ project-api-test-jobs ]
+    if: ${{ contains( needs.*.result, 'success' ) || contains( needs.*.result, 'failure' ) }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 'Install Allure CLI'
+        env:
+          DESTINATION_PATH: ../
+        run: ./.github/workflows/scripts/install-allure.sh
+
+      - name: 'Download blob reports from artifacts'
+        uses: actions/download-artifact@v4
+        with:
+          path: ./out/allure-results
+          pattern: all-blob-reports-*
+          run-id: project-api-test-jobs
+          merge-multiple: true
+
+      - name: 'Generate Allure report'
+        id: generate_allure_report
+        run: allure generate --clean ./out/allure-results --output ./out/allure-report
+
+      - name: 'Archive reports'
+        uses: actions/upload-artifact@v4
+        with:
+          name: api-test-report
+          path: ./out
+          if-no-files-found: ignore
+          retention-days: 5
+          
+      - name: 'Publish reports'
+        env:
+          GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          RUN_ID: ${{ github.run_id }}
+          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          if [ "$GITHUB_EVENT_NAME" == pull_request ]; then
+            gh workflow run publish-test-reports-pr.yml \
+              -f run_id=$RUN_ID \
+              -f api_artifact=api-test-report \
+              -f pr_number=$PR_NUMBER \
+              -f commit_sha=$COMMIT_SHA \
+              -f s3_root=public \
+              --repo woocommerce/woocommerce-test-reports
+          else
+            gh workflow run publish-test-reports-trunk-merge.yml \
+              -f run_id=$RUN_ID \
+              -f artifact=api-test-report \
+              -f pr_number='' \
+              -f test_type="api" \
+              --repo woocommerce/woocommerce-test-reports
+          fi

.github/workflows/pr-build-and-e2e-tests.yml

@@ -1,9 +1,9 @@
 name: Run tests against PR
 on:
     workflow_dispatch:
-    pull_request:
+    #pull_request:
-        paths-ignore:
+        #paths-ignore:
-            - '**/changelog/**'
+            #- '**/changelog/**'
 
 concurrency:
     group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/scripts/install-allure.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Test Java installation
+java -version
+
+if [[ -z "$DESTINATION_PATH" ]]; then
+	echo "::error::DESTINATION_PATH must be set"
+	exit 1
+fi
+
+ALLURE_VERSION=2.27.0
+ALLURE_DOWNLOAD_URL=https://github.com/allure-framework/allure2/releases/download/$ALLURE_VERSION/allure-$ALLURE_VERSION.zip
+
+echo "Installing Allure $ALLURE_VERSION in $DESTINATION_PATH"
+wget --no-verbose -O allure.zip $ALLURE_DOWNLOAD_URL \
+  && unzip allure.zip -d "$DESTINATION_PATH" \
+  && rm -rf allure.zip \
+
+ALLURE_PATH=$(realpath "$DESTINATION_PATH"/allure-$ALLURE_VERSION/bin)
+
+# Test Allure installation
+echo "$ALLURE_PATH"
+export PATH="$ALLURE_PATH:$PATH"
+allure --version
+
+# Add Allure in Github PATH to make it available to all subsequent actions in the current job
+echo "$ALLURE_PATH" >> "$GITHUB_PATH"

.github/workflows/smoke-test-pr-merge.yml

@@ -1,8 +1,9 @@
 name: Run tests against trunk after PR merge
 on:
-    pull_request:
+  workflow_dispatch: 
-        types:
+    #pull_request:
-            - closed
+        #types:
+            #- closed
 concurrency:
     group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
     cancel-in-progress: true

plugins/woocommerce/changelog/tools-add-e2e-tests-in-ci-jobs

@@ -0,0 +1,4 @@
+Significance: minor
+Type: dev
+
+CI: adds e2e tests into ci-jobs and ci.yml

plugins/woocommerce/package.json

@@ -28,8 +28,10 @@
 		"env:restart": "pnpm wp-env destroy && pnpm wp-env start --update",
 		"env:start": "pnpm wp-env start",
 		"env:stop": "pnpm wp-env stop",
-		"env:test": "WP_ENV_LIFECYCLE_SCRIPT_AFTER_START='./tests/e2e-pw/bin/test-env-setup.sh' && pnpm env:dev",
+		"env:test": "WP_ENV_LIFECYCLE_SCRIPT_AFTER_START='./tests/e2e-pw/bin/test-env-setup.sh' && pnpm env:dev && pnpm playwright install chromium",
 		"env:test:cot": "WP_ENV_LIFECYCLE_SCRIPT_AFTER_START='ENABLE_HPOS=1 ./tests/e2e-pw/bin/test-env-setup.sh' && ENABLE_HPOS=1 pnpm env:dev",
+		"env:perf:install-k6": "curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1",
+		"env:perf": "pnpm env:dev && pnpm env:performance-init && pnpm env:perf:install-k6",
 		"preinstall": "npx only-allow pnpm",
 		"postinstall": "composer install",
 		"lint": "pnpm --if-present '/^lint:lang:.*$/'",
@@ -48,6 +50,7 @@
 		"test:api": "API_TEST_REPORT_DIR=\"$PWD/tests/api\" pnpm exec wc-api-tests test api",
 		"test:api-pw": "USE_WP_ENV=1 pnpm playwright test --config=tests/api-core-tests/playwright.config.js",
 		"test:e2e-pw": "USE_WP_ENV=1 pnpm playwright test --config=tests/e2e-pw/playwright.config.js",
+		"test:perf": "./k6 run ./tests/performance/tests/gh-action-pr-requests.js",
 		"test:env:start": "pnpm env:test",
 		"test:php": "./vendor/bin/phpunit -c ./phpunit.xml",
 		"test:php:watch": "./vendor/bin/phpunit-watcher watch",
@@ -165,6 +168,40 @@
 							"wpVersion": "latest-2"
 						}
 					}
+				},
+				{
+					"name": "Core e2e tests",
+					"testType": "e2e",
+					"command": "test:e2e-pw",
+					"shardingArguments": [
+						"--shard=1/5",
+						"--shard=2/5", 
+						"--shard=3/5", 
+						"--shard=4/5", 
+						"--shard=5/5"
+					],
+					"changes": [],
+					"testEnv": {
+						"start": "env:test"
+					}
+				},
+				{
+					"name": "Core API tests",
+					"testType": "api",
+					"command": "test:api-pw",
+					"changes": [],
+					"testEnv": {
+						"start": "env:test"
+					}
+				},
+				{
+					"name": "Core Performance tests (K6)",
+					"testType": "performance",
+					"command": "test:perf",
+					"changes": [],
+					"testEnv": {
+						"start": "env:perf"
+					}
 				}
 			]
 		}

tools/monorepo-utils/src/ci-jobs/index.ts

@@ -12,6 +12,7 @@ import { buildProjectGraph } from './lib/project-graph';
 import { getFileChanges } from './lib/file-changes';
 import { createJobsForChanges } from './lib/job-processing';
 import { isGithubCI } from '../core/environment';
+import { testTypes } from './lib/config';
 
 const program = new Command( 'ci-jobs' )
 	.description(
@@ -49,7 +50,13 @@ const program = new Command( 'ci-jobs' )
 
 		if ( isGithubCI() ) {
 			setOutput( 'lint-jobs', JSON.stringify( jobs.lint ) );
-			setOutput( 'test-jobs', JSON.stringify( jobs.test ) );
+
+			testTypes.forEach( ( type ) => {
+				setOutput(
+					`${ type }-test-jobs`,
+					JSON.stringify( jobs[ `${ type }Test` ] )
+				);
+			} );
 			return;
 		}
 
@@ -62,14 +69,16 @@ const program = new Command( 'ci-jobs' )
 			Logger.notice( 'No lint jobs to run.' );
 		}
 
-		if ( jobs.test.length > 0 ) {
+		testTypes.forEach( ( type ) => {
-			Logger.notice( 'Test Jobs' );
+			if ( jobs[ `${ type }Test` ].length > 0 ) {
-			for ( const job of jobs.test ) {
+				Logger.notice( `${ type } test Jobs` );
+				for ( const job of jobs[ `${ type }Test` ] ) {
 					Logger.notice( `-  ${ job.projectName } - ${ job.name }` );
 				}
 			} else {
-			Logger.notice( 'No test jobs to run.' );
+				Logger.notice( `No ${ type } test jobs to run.` );
 			}
 		} );
+	} );
 
 export default program;

tools/monorepo-utils/src/ci-jobs/lib/__tests__/config.spec.ts

@@ -6,7 +6,7 @@ import { makeRe } from 'minimatch';
 /**
  * Internal dependencies
  */
-import { JobType, parseCIConfig } from '../config';
+import { JobType, parseCIConfig, testTypes } from '../config';
 
 describe( 'Config', () => {
 	describe( 'parseCIConfig', () => {
@@ -136,6 +136,8 @@ describe( 'Config', () => {
 				jobs: [
 					{
 						type: JobType.Test,
+						testType: 'default',
+						shardingArguments: [],
 						name: 'default',
 						changes: [
 							/^package\.json$/,
@@ -222,5 +224,129 @@ describe( 'Config', () => {
 				],
 			} );
 		} );
+
+		it.each( testTypes )(
+			'should parse test config with expected testType',
+			( testType ) => {
+				const parsed = parseCIConfig( {
+					name: 'foo',
+					config: {
+						ci: {
+							tests: [
+								{
+									name: 'default',
+									testType,
+									changes: '/src/**/*.{js,jsx,ts,tsx}',
+									command: 'foo',
+								},
+							],
+						},
+					},
+				} );
+
+				expect( parsed ).toMatchObject( {
+					jobs: [
+						{
+							type: JobType.Test,
+							testType,
+							shardingArguments: [],
+							name: 'default',
+							changes: [
+								/^package\.json$/,
+								makeRe( '/src/**/*.{js,jsx,ts,tsx}' ),
+							],
+							command: 'foo',
+						},
+					],
+				} );
+			}
+		);
+
+		it.each( [
+			[ '', 'default' ],
+			[ 'bad', 'default' ],
+			[ 1, 'default' ],
+			[ undefined, 'default' ],
+		] )(
+			'should parse test config with unexpected testType',
+			( input, result ) => {
+				const parsed = parseCIConfig( {
+					name: 'foo',
+					config: {
+						ci: {
+							tests: [
+								{
+									name: 'default',
+									testType: input,
+									changes: '/src/**/*.{js,jsx,ts,tsx}',
+									command: 'foo',
+								},
+							],
+						},
+					},
+				} );
+
+				expect( parsed ).toMatchObject( {
+					jobs: [
+						{
+							type: JobType.Test,
+							testType: result,
+							shardingArguments: [],
+							name: 'default',
+							changes: [
+								/^package\.json$/,
+								makeRe( '/src/**/*.{js,jsx,ts,tsx}' ),
+							],
+							command: 'foo',
+						},
+					],
+				} );
+			}
+		);
+
+		it.each( [
+			[ [], [] ],
+			[ undefined, [] ],
+			[
+				[ 'a', 'b' ],
+				[ 'a', 'b' ],
+			],
+		] )(
+			'should parse test config with shards',
+			( shardingArguments: any, result: string[] ) => {
+				const parsed = parseCIConfig( {
+					name: 'foo',
+					config: {
+						ci: {
+							tests: [
+								{
+									name: 'default',
+									testType: 'e2e',
+									shardingArguments,
+									changes: '/src/**/*.{js,jsx,ts,tsx}',
+									command: 'foo',
+								},
+							],
+						},
+					},
+				} );
+
+				expect( parsed ).toMatchObject( {
+					jobs: [
+						{
+							type: JobType.Test,
+							testType: 'e2e',
+							shardingArguments: result,
+							name: 'default',
+							changes: [
+								/^package\.json$/,
+								makeRe( '/src/**/*.{js,jsx,ts,tsx}' ),
+							],
+							command: 'foo',
+						},
+					],
+				} );
+			}
+		);
 	} );
 } );

tools/monorepo-utils/src/ci-jobs/lib/__tests__/job-processing.spec.ts

@@ -1,8 +1,8 @@
 /**
  * Internal dependencies
  */
-import { JobType } from '../config';
+import { JobType, testTypes } from '../config';
-import { createJobsForChanges } from '../job-processing';
+import { createJobsForChanges, getShardedJobs } from '../job-processing';
 import { parseTestEnvConfig } from '../test-environment';
 
 jest.mock( '../test-environment' );
@@ -49,6 +49,7 @@ describe( 'Job Processing', () => {
 			expect( jobs.lint ).toHaveLength( 1 );
 			expect( jobs.lint ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				command: 'test-lint',
 			} );
 			expect( jobs.test ).toHaveLength( 0 );
@@ -83,6 +84,7 @@ describe( 'Job Processing', () => {
 			expect( jobs.lint ).toHaveLength( 1 );
 			expect( jobs.lint ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				command: 'test-lint test-base-ref',
 			} );
 			expect( jobs.test ).toHaveLength( 0 );
@@ -220,10 +222,12 @@ describe( 'Job Processing', () => {
 			expect( jobs.lint ).toHaveLength( 2 );
 			expect( jobs.lint ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				command: 'test-lint',
 			} );
 			expect( jobs.lint ).toContainEqual( {
 				projectName: 'test-b',
+				projectPath: 'test-b',
 				command: 'test-lint-b',
 			} );
 			expect( jobs.test ).toHaveLength( 0 );
@@ -276,16 +280,20 @@ describe( 'Job Processing', () => {
 			expect( jobs.lint ).toHaveLength( 2 );
 			expect( jobs.lint ).toContainEqual( {
 				projectName: 'test-a',
+				projectPath: 'test-a',
 				command: 'test-lint-a',
 			} );
 			expect( jobs.lint ).toContainEqual( {
 				projectName: 'test-b',
+				projectPath: 'test-b',
 				command: 'test-lint-b',
 			} );
 			expect( jobs.test ).toHaveLength( 0 );
 		} );
 
 		it( 'should trigger test job for single node', async () => {
+			const testType = 'default';
+
 			const jobs = await createJobsForChanges(
 				{
 					name: 'test',
@@ -294,6 +302,8 @@ describe( 'Job Processing', () => {
 						jobs: [
 							{
 								type: JobType.Test,
+								testType,
+								shardingArguments: [],
 								name: 'Default',
 								changes: [ /test.js$/ ],
 								command: 'test-cmd',
@@ -309,11 +319,13 @@ describe( 'Job Processing', () => {
 			);
 
 			expect( jobs.lint ).toHaveLength( 0 );
-			expect( jobs.test ).toHaveLength( 1 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				name: 'Default',
 				command: 'test-cmd',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: false,
 					envVars: {},
@@ -322,6 +334,7 @@ describe( 'Job Processing', () => {
 		} );
 
 		it( 'should replace vars in test command', async () => {
+			const testType = 'default';
 			const jobs = await createJobsForChanges(
 				{
 					name: 'test',
@@ -330,7 +343,9 @@ describe( 'Job Processing', () => {
 						jobs: [
 							{
 								type: JobType.Test,
+								testType,
 								name: 'Default',
+								shardingArguments: [],
 								changes: [ /test.js$/ ],
 								command: 'test-cmd <baseRef>',
 							},
@@ -349,11 +364,13 @@ describe( 'Job Processing', () => {
 			);
 
 			expect( jobs.lint ).toHaveLength( 0 );
-			expect( jobs.test ).toHaveLength( 1 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				name: 'Default',
 				command: 'test-cmd test-base-ref',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: false,
 					envVars: {},
@@ -362,6 +379,7 @@ describe( 'Job Processing', () => {
 		} );
 
 		it( 'should not trigger a test job that has already been created', async () => {
+			const testType = 'default';
 			const jobs = await createJobsForChanges(
 				{
 					name: 'test',
@@ -370,7 +388,9 @@ describe( 'Job Processing', () => {
 						jobs: [
 							{
 								type: JobType.Test,
+								testType,
 								name: 'Default',
+								shardingArguments: [],
 								changes: [ /test.js$/ ],
 								command: 'test-cmd',
 								jobCreated: true,
@@ -386,10 +406,11 @@ describe( 'Job Processing', () => {
 			);
 
 			expect( jobs.lint ).toHaveLength( 0 );
-			expect( jobs.test ).toHaveLength( 0 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 0 );
 		} );
 
 		it( 'should not trigger test job for single node with no changes', async () => {
+			const testType = 'default';
 			const jobs = await createJobsForChanges(
 				{
 					name: 'test',
@@ -398,7 +419,9 @@ describe( 'Job Processing', () => {
 						jobs: [
 							{
 								type: JobType.Test,
+								testType,
 								name: 'Default',
+								shardingArguments: [],
 								changes: [ /test.js$/ ],
 								command: 'test-cmd',
 							},
@@ -411,10 +434,11 @@ describe( 'Job Processing', () => {
 			);
 
 			expect( jobs.lint ).toHaveLength( 0 );
-			expect( jobs.test ).toHaveLength( 0 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 0 );
 		} );
 
 		it( 'should trigger test job for project graph', async () => {
+			const testType = 'default';
 			const jobs = await createJobsForChanges(
 				{
 					name: 'test',
@@ -423,7 +447,9 @@ describe( 'Job Processing', () => {
 						jobs: [
 							{
 								type: JobType.Test,
+								testType,
 								name: 'Default',
+								shardingArguments: [],
 								changes: [ /test.js$/ ],
 								command: 'test-cmd',
 							},
@@ -437,7 +463,9 @@ describe( 'Job Processing', () => {
 								jobs: [
 									{
 										type: JobType.Test,
+										testType: 'default',
 										name: 'Default A',
+										shardingArguments: [],
 										changes: [ /test-b.js$/ ],
 										command: 'test-cmd-a',
 									},
@@ -452,7 +480,9 @@ describe( 'Job Processing', () => {
 								jobs: [
 									{
 										type: JobType.Test,
+										testType: 'default',
 										name: 'Default B',
+										shardingArguments: [],
 										changes: [ /test-b.js$/ ],
 										command: 'test-cmd-b',
 									},
@@ -471,20 +501,24 @@ describe( 'Job Processing', () => {
 			);
 
 			expect( jobs.lint ).toHaveLength( 0 );
-			expect( jobs.test ).toHaveLength( 2 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 2 );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				name: 'Default',
 				command: 'test-cmd',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: false,
 					envVars: {},
 				},
 			} );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test-b',
+				projectPath: 'test-b',
 				name: 'Default B',
 				command: 'test-cmd-b',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: false,
 					envVars: {},
@@ -492,7 +526,9 @@ describe( 'Job Processing', () => {
 			} );
 		} );
 
-		it( 'should trigger test job for dependent without changes when dependency has matching cascade key', async () => {
+		it.each( testTypes )(
+			'should trigger %s test job for single node',
+			async ( testType ) => {
 				const jobs = await createJobsForChanges(
 					{
 						name: 'test',
@@ -501,7 +537,51 @@ describe( 'Job Processing', () => {
 							jobs: [
 								{
 									type: JobType.Test,
+									testType,
 									name: 'Default',
+									shardingArguments: [],
+									changes: [ /test.js$/ ],
+									command: 'test-cmd',
+								},
+							],
+						},
+						dependencies: [],
+					},
+					{
+						test: [ 'test.js' ],
+					},
+					{}
+				);
+
+				expect( jobs.lint ).toHaveLength( 0 );
+				expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
+				expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
+					projectName: 'test',
+					projectPath: 'test',
+					name: 'Default',
+					command: 'test-cmd',
+					shardNumber: 0,
+					testEnv: {
+						shouldCreate: false,
+						envVars: {},
+					},
+				} );
+			}
+		);
+
+		it( 'should trigger test job for dependent without changes when dependency has matching cascade key', async () => {
+			const testType = 'default';
+			const jobs = await createJobsForChanges(
+				{
+					name: 'test',
+					path: 'test',
+					ciConfig: {
+						jobs: [
+							{
+								type: JobType.Test,
+								testType,
+								name: 'Default',
+								shardingArguments: [],
 								changes: [ /test.js$/ ],
 								command: 'test-cmd',
 								cascadeKeys: [ 'test' ],
@@ -516,7 +596,9 @@ describe( 'Job Processing', () => {
 								jobs: [
 									{
 										type: JobType.Test,
+										testType: 'default',
 										name: 'Default A',
+										shardingArguments: [],
 										changes: [ /test-a.js$/ ],
 										command: 'test-cmd-a',
 										cascadeKeys: [ 'test-a', 'test' ],
@@ -534,20 +616,24 @@ describe( 'Job Processing', () => {
 			);
 
 			expect( jobs.lint ).toHaveLength( 0 );
-			expect( jobs.test ).toHaveLength( 2 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 2 );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				name: 'Default',
 				command: 'test-cmd',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: false,
 					envVars: {},
 				},
 			} );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test-a',
+				projectPath: 'test-a',
 				name: 'Default A',
 				command: 'test-cmd-a',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: false,
 					envVars: {},
@@ -556,6 +642,7 @@ describe( 'Job Processing', () => {
 		} );
 
 		it( 'should isolate dependency cascade keys to prevent cross-dependency matching', async () => {
+			const testType = 'default';
 			const jobs = await createJobsForChanges(
 				{
 					name: 'test',
@@ -564,7 +651,9 @@ describe( 'Job Processing', () => {
 						jobs: [
 							{
 								type: JobType.Test,
+								testType,
 								name: 'Default',
+								shardingArguments: [],
 								changes: [ /test.js$/ ],
 								command: 'test-cmd',
 								cascadeKeys: [ 'test' ],
@@ -579,7 +668,9 @@ describe( 'Job Processing', () => {
 								jobs: [
 									{
 										type: JobType.Test,
+										testType: 'default',
 										name: 'Default A',
+										shardingArguments: [],
 										changes: [ /test-a.js$/ ],
 										command: 'test-cmd-a',
 										cascadeKeys: [ 'test-a', 'test' ],
@@ -595,7 +686,9 @@ describe( 'Job Processing', () => {
 								jobs: [
 									{
 										type: JobType.Test,
+										testType: 'default',
 										name: 'Default B',
+										shardingArguments: [],
 										changes: [ /test-b.js$/ ],
 										command: 'test-cmd-b',
 										cascadeKeys: [ 'test-b', 'test' ],
@@ -613,20 +706,24 @@ describe( 'Job Processing', () => {
 			);
 
 			expect( jobs.lint ).toHaveLength( 0 );
-			expect( jobs.test ).toHaveLength( 2 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 2 );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				name: 'Default',
 				command: 'test-cmd',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: false,
 					envVars: {},
 				},
 			} );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test-a',
+				projectPath: 'test-a',
 				name: 'Default A',
 				command: 'test-cmd-a',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: false,
 					envVars: {},
@@ -635,6 +732,7 @@ describe( 'Job Processing', () => {
 		} );
 
 		it( 'should trigger test job for single node and parse test environment config', async () => {
+			const testType = 'default';
 			jest.mocked( parseTestEnvConfig ).mockResolvedValue( {
 				WP_ENV_CORE: 'https://wordpress.org/latest.zip',
 			} );
@@ -647,7 +745,9 @@ describe( 'Job Processing', () => {
 						jobs: [
 							{
 								type: JobType.Test,
+								testType,
 								name: 'Default',
+								shardingArguments: [],
 								changes: [ /test.js$/ ],
 								command: 'test-cmd',
 								testEnv: {
@@ -672,11 +772,13 @@ describe( 'Job Processing', () => {
 			);
 
 			expect( jobs.lint ).toHaveLength( 0 );
-			expect( jobs.test ).toHaveLength( 1 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				name: 'Default',
 				command: 'test-cmd',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: true,
 					start: 'test-start test-base-ref',
@@ -688,6 +790,7 @@ describe( 'Job Processing', () => {
 		} );
 
 		it( 'should trigger all jobs for a single node with changes set to "true"', async () => {
+			const testType = 'default';
 			const jobs = await createJobsForChanges(
 				{
 					name: 'test',
@@ -701,7 +804,9 @@ describe( 'Job Processing', () => {
 							},
 							{
 								type: JobType.Test,
+								testType,
 								name: 'Default',
+								shardingArguments: [],
 								changes: [ /test.js$/ ],
 								command: 'test-cmd',
 							},
@@ -716,18 +821,174 @@ describe( 'Job Processing', () => {
 			expect( jobs.lint ).toHaveLength( 1 );
 			expect( jobs.lint ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				command: 'test-lint',
 			} );
-			expect( jobs.test ).toHaveLength( 1 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 1 );
-			expect( jobs.test ).toContainEqual( {
+			expect( jobs[ `${ testType }Test` ] ).toContainEqual( {
 				projectName: 'test',
+				projectPath: 'test',
 				name: 'Default',
 				command: 'test-cmd',
+				shardNumber: 0,
 				testEnv: {
 					shouldCreate: false,
 					envVars: {},
 				},
 			} );
 		} );
+
+		it( 'should trigger sharded test jobs for single node', async () => {
+			const testType = 'default';
+			const jobs = await createJobsForChanges(
+				{
+					name: 'test',
+					path: 'test',
+					ciConfig: {
+						jobs: [
+							{
+								type: JobType.Test,
+								testType,
+								name: 'Default',
+								shardingArguments: [
+									'--shard=1/2',
+									'--shard=2/2',
+								],
+								changes: [ /test.js$/ ],
+								command: 'test-cmd',
+							},
+						],
+					},
+					dependencies: [],
+				},
+				{
+					test: [ 'test.js' ],
+				},
+				{}
+			);
+
+			expect( jobs.lint ).toHaveLength( 0 );
+			expect( jobs[ `${ testType }Test` ] ).toHaveLength( 2 );
+			expect( jobs[ `${ testType }Test` ] ).toEqual(
+				expect.arrayContaining( [
+					{
+						projectName: 'test',
+						projectPath: 'test',
+						name: 'Default 1/2',
+						command: 'test-cmd --shard=1/2',
+						shardNumber: 1,
+						testEnv: {
+							shouldCreate: false,
+							envVars: {},
+						},
+					},
+					{
+						projectName: 'test',
+						projectPath: 'test',
+						name: 'Default 2/2',
+						command: 'test-cmd --shard=2/2',
+						shardNumber: 2,
+						testEnv: {
+							shouldCreate: false,
+							envVars: {},
+						},
+					},
+				] )
+			);
+		} );
+	} );
+
+	describe( 'getShardedJobs', () => {
+		it( 'should create sharded jobs', async () => {
+			const jobs = getShardedJobs(
+				{
+					projectName: 'test',
+					projectPath: 'test',
+					name: 'Default',
+					command: 'test-cmd',
+					shardNumber: 0,
+					testEnv: {
+						shouldCreate: false,
+						envVars: {},
+					},
+				},
+				{
+					type: JobType.Test,
+					testType: 'e2e',
+					name: 'Default',
+					shardingArguments: [ '--shard-arg-1', '--shard-arg-2' ],
+					changes: [ /test.js$/ ],
+					command: 'test-cmd',
+				}
+			);
+
+			expect( jobs ).toHaveLength( 2 );
+			expect( jobs ).toEqual(
+				expect.arrayContaining( [
+					{
+						projectName: 'test',
+						projectPath: 'test',
+						name: 'Default 1/2',
+						command: 'test-cmd --shard-arg-1',
+						shardNumber: 1,
+						testEnv: {
+							shouldCreate: false,
+							envVars: {},
+						},
+					},
+					{
+						projectName: 'test',
+						projectPath: 'test',
+						name: 'Default 2/2',
+						command: 'test-cmd --shard-arg-2',
+						shardNumber: 2,
+						testEnv: {
+							shouldCreate: false,
+							envVars: {},
+						},
+					},
+				] )
+			);
+		} );
+
+		it.each( [ [ [] ], [ [ '--sharding=1/1' ] ] ] )(
+			'should not create sharded jobs for shards',
+			async ( shardingArguments ) => {
+				const jobs = getShardedJobs(
+					{
+						projectName: 'test',
+						projectPath: 'test',
+						name: 'Default',
+						command: 'test-cmd',
+						shardNumber: 0,
+						testEnv: {
+							shouldCreate: false,
+							envVars: {},
+						},
+					},
+					{
+						type: JobType.Test,
+						testType: 'e2e',
+						name: 'Default',
+						shardingArguments,
+						changes: [ /test.js$/ ],
+						command: 'test-cmd',
+					}
+				);
+
+				expect( jobs ).toHaveLength( 1 );
+				expect( jobs ).toContainEqual( {
+					projectName: 'test',
+					projectPath: 'test',
+					name: 'Default',
+					command: 'test-cmd',
+					shardNumber: 0,
+					testEnv: {
+						shouldCreate: false,
+						envVars: {},
+					},
+				} );
+			}
+		);
 	} );
 } );

tools/monorepo-utils/src/ci-jobs/lib/config.ts

@@ -21,6 +21,11 @@ export const enum JobType {
 	Test = 'test',
 }
 
+/**
+ * The type of the test job.
+ */
+export const testTypes = [ 'default', 'e2e', 'api', 'performance' ] as const;
+
 /**
  * The variables that can be used in tokens on command strings
  * that will be replaced during job creation.
@@ -251,11 +256,21 @@ export interface TestJobConfig extends BaseJobConfig {
 	 */
 	type: JobType.Test;
 
+	/**
+	 * The type of the test.
+	 */
+	testType: ( typeof testTypes )[ number ];
+
 	/**
 	 * The name for the job.
 	 */
 	name: string;
 
+	/**
+	 * The number of shards to be created for this job.
+	 */
+	shardingArguments: string[];
+
 	/**
 	 * The configuration for the test environment if one is needed.
 	 */
@@ -320,10 +335,20 @@ function parseTestJobConfig( raw: any ): TestJobConfig {
 		);
 	}
 
+	let testType: ( typeof testTypes )[ number ] = 'default';
+	if (
+		raw.testType &&
+		testTypes.includes( raw.testType.toString().toLowerCase() )
+	) {
+		testType = raw.testType.toLowerCase();
+	}
+
 	validateCommandVars( raw.command );
 
 	const config: TestJobConfig = {
 		type: JobType.Test,
+		testType,
+		shardingArguments: raw.shardingArguments || [],
 		name: raw.name,
 		changes: parseChangesConfig( raw.changes, [ 'package.json' ] ),
 		command: raw.command,

tools/monorepo-utils/src/ci-jobs/lib/job-processing.ts

@@ -4,6 +4,7 @@
 import {
 	CommandVarOptions,
 	JobType,
+	testTypes,
 	LintJobConfig,
 	TestJobConfig,
 } from './config';
@@ -16,6 +17,7 @@ import { TestEnvVars, parseTestEnvConfig } from './test-environment';
  */
 interface LintJob {
 	projectName: string;
+	projectPath: string;
 	command: string;
 }
 
@@ -33,9 +35,11 @@ interface TestJobEnv {
  */
 interface TestJob {
 	projectName: string;
+	projectPath: string;
 	name: string;
 	command: string;
 	testEnv: TestJobEnv;
+	shardNumber: number;
 }
 
 /**
@@ -70,10 +74,42 @@ function replaceCommandVars( command: string, options: CreateOptions ): string {
 	} );
 }
 
+/**
+ * Multiplies a job based on the shards job config. It updates the job names and command - currently only supporting Playwright sharding.
+ *
+ * @param {TestJob}       job       The job to be multiplied.
+ * @param {TestJobConfig} jobConfig The job config.
+ * @return {TestJob[]} The list of sharded jobs.
+ */
+export function getShardedJobs(
+	job: TestJob,
+	jobConfig: TestJobConfig
+): TestJob[] {
+	let createdJobs = [];
+	const shards = jobConfig.shardingArguments.length;
+
+	if ( shards <= 1 ) {
+		createdJobs.push( job );
+	} else {
+		createdJobs = Array( shards )
+			.fill( null )
+			.map( ( _, i ) => {
+				const jobCopy = JSON.parse( JSON.stringify( job ) );
+				jobCopy.shardNumber = i + 1;
+				jobCopy.name = `${ job.name } ${ i + 1 }/${ shards }`;
+				jobCopy.command = `${ job.command } ${ jobConfig.shardingArguments[ i ] }`;
+				return jobCopy;
+			} );
+	}
+
+	return createdJobs;
+}
+
 /**
  * Checks the config against the changes and creates one if it should be run.
  *
  * @param {string}              projectName The name of the project that the job is for.
+ * @param {string}              projectPath The path of the project that the job is for.
  * @param {Object}              config      The config object for the lint job.
  * @param {Array.<string>|true} changes     The file changes that have occurred for the project or true if all projects should be marked as changed.
  * @param {Object}              options     The options to use when creating the job.
@@ -81,6 +117,7 @@ function replaceCommandVars( command: string, options: CreateOptions ): string {
  */
 function createLintJob(
 	projectName: string,
+	projectPath: string,
 	config: LintJobConfig,
 	changes: string[] | true,
 	options: CreateOptions
@@ -114,6 +151,7 @@ function createLintJob(
 
 	return {
 		projectName,
+		projectPath,
 		command: replaceCommandVars( config.command, options ),
 	};
 }
@@ -122,18 +160,22 @@ function createLintJob(
  * Checks the config against the changes and creates one if it should be run.
  *
  * @param {string}              projectName The name of the project that the job is for.
+ * @param {string}              projectPath The path of the project that the job is for.
  * @param {Object}              config      The config object for the test job.
  * @param {Array.<string>|true} changes     The file changes that have occurred for the project or true if all projects should be marked as changed.
  * @param {Object}              options     The options to use when creating the job.
  * @param {Array.<string>}      cascadeKeys The cascade keys that have been triggered in dependencies.
+ * @param {number}              shardNumber The shard number for the job.
  * @return {Promise.<Object|null>} The job that should be run or null if no job should be run.
  */
 async function createTestJob(
 	projectName: string,
+	projectPath: string,
 	config: TestJobConfig,
 	changes: string[] | true,
 	options: CreateOptions,
-	cascadeKeys: string[]
+	cascadeKeys: string[],
+	shardNumber: number
 ): Promise< TestJob | null > {
 	let triggered = false;
 
@@ -179,12 +221,14 @@ async function createTestJob(
 
 	const createdJob: TestJob = {
 		projectName,
+		projectPath,
 		name: config.name,
 		command: replaceCommandVars( config.command, options ),
 		testEnv: {
 			shouldCreate: false,
 			envVars: {},
 		},
+		shardNumber,
 	};
 
 	// We want to make sure that we're including the configuration for
@@ -221,6 +265,10 @@ async function createJobsForProject(
 		test: [],
 	};
 
+	testTypes.forEach( ( type ) => {
+		newJobs[ `${ type }Test` ] = [];
+	} );
+
 	// In order to simplify the way that cascades work we're going to recurse depth-first and check our dependencies
 	// for jobs before ourselves. This lets any cascade keys created in dependencies cascade to dependents.
 	const newCascadeKeys = [];
@@ -240,7 +288,12 @@ async function createJobsForProject(
 			dependencyCascade
 		);
 		newJobs.lint.push( ...dependencyJobs.lint );
-		newJobs.test.push( ...dependencyJobs.test );
+
+		testTypes.forEach( ( type ) => {
+			newJobs[ `${ type }Test` ].push(
+				...dependencyJobs[ `${ type }Test` ]
+			);
+		} );
 
 		// Track any new cascade keys added by the dependency.
 		// Since we're filtering out duplicates after the
@@ -285,6 +338,7 @@ async function createJobsForProject(
 			case JobType.Lint: {
 				const created = createLintJob(
 					node.name,
+					node.path,
 					jobConfig,
 					projectChanges,
 					options
@@ -301,17 +355,22 @@ async function createJobsForProject(
 			case JobType.Test: {
 				const created = await createTestJob(
 					node.name,
+					node.path,
 					jobConfig,
 					projectChanges,
 					options,
-					cascadeKeys
+					cascadeKeys,
+					0
 				);
 				if ( ! created ) {
 					break;
 				}
 
 				jobConfig.jobCreated = true;
-				newJobs.test.push( created );
+
+				newJobs[ `${ jobConfig.testType }Test` ].push(
+					...getShardedJobs( created, jobConfig )
+				);
 
 				// We need to track any cascade keys that this job is associated with so that
 				// dependent projects can trigger jobs with matching keys. We are expecting