[blocks e2e] Move Blocks e2e into ci.yml (#48224)

.github/workflows/ci.yml

@@ -163,14 +163,26 @@ jobs:
                 CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} # required by Metrics tests
               run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
               
+            - name: 'Resolve artifacts path'
+              # Blocks e2e use a relative path which is not supported by actions/upload-artifact@v4
+              # https://github.com/actions/upload-artifact/issues/176
+              env:
+                ARTIFACTS_PATH: '${{ matrix.projectPath }}/${{ matrix.report.resultsPath }}'
+              run: echo "ARTIFACTS_PATH=$(realpath $ARTIFACTS_PATH)" >> $GITHUB_ENV
+              
             - name: 'Upload artifacts'
               if: ${{ always() && matrix.report.resultsPath != '' }}
               uses: actions/upload-artifact@v4
               with:
                 name: '${{ matrix.report.resultsBlobName }}-${{ strategy.job-index }}'
-                path: '${{ matrix.projectPath }}/${{ matrix.report.resultsPath }}'
+                path: ${{ env.ARTIFACTS_PATH }}
-                retention-days: 1
+                
-                compression-level: 9
+            - name: 'Upload flaky test reports'
+              uses: actions/upload-artifact@v4
+              with:
+                name: flaky-tests-${{ strategy.job-index }}
+                path: ${{ env.ARTIFACTS_PATH }}/flaky-tests
+                if-no-files-found: ignore
 
             - name: 'Archive metrics results'
               if: ${{ success() && startsWith(matrix.name, 'Metrics') }} # this seems too fragile, we should update the reporting path and use the generic upload step above
@@ -338,3 +350,49 @@ jobs:
               if: ${{ github.event_name == 'pull_request' && contains(matrix.report, 'e2e') }}
               run: |
                   echo "::notice::🔗🔗 The e2e report for this run is available at https://woocommerce.github.io/woocommerce-test-reports/pr/${{ github.event.pull_request.number }}/e2e"
+
+
+    report-flaky-tests:
+        name: 'Create issues for flaky tests'
+        if: ${{ !cancelled() && ! github.event.pull_request.head.repo.fork }}
+        needs: ['project-test-jobs']
+        runs-on: ubuntu-latest
+        permissions:
+          contents: read
+          issues: write
+
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  repository: WordPress/gutenberg
+                  ref: dbf201449e9736f672b61e422787d47659db327a
+                  
+            - uses: actions/download-artifact@v4
+              id: download-artifact
+              with:
+                pattern: flaky-tests*
+                path: flaky-tests
+                merge-multiple: true
+                
+            - name: 'Check if there are flaky tests reports'
+              run: |
+                downloadPath='${{ steps.download-artifact.outputs.download-path }}'
+                # make dir so that next step doesn't fail if it doesn't exist 
+                mkdir -p $downloadPath
+                # any output means there are reports
+                echo "FLAKY_REPORTS=$(ls -A $downloadPath | head -1)" >> $GITHUB_ENV
+                
+            - name: 'Setup'
+              if: ${{ !!env.FLAKY_REPORTS }}
+              uses: ./.github/setup-node
+
+            - name: 'Build packages'
+              if: ${{ !!env.FLAKY_REPORTS }}
+              run: npm run build:packages
+
+            - name: 'Report flaky tests'
+              if: ${{ !!env.FLAKY_REPORTS }}
+              uses: ./packages/report-flaky-tests
+              with:
+                  repo-token: ${{ secrets.GITHUB_TOKEN }}
+                  label: 'metric: flaky e2e test'

.github/workflows/scripts/evaluate-jobs-conclusions.js

@@ -4,6 +4,7 @@ const IGNORED_JOBS = [
 	/Evaluate Project Job Statuses/,
 	/Report results on Slack/,
 	/Publish reports/,
+	/Create issues for flaky tests/,
 ];
 
 const isJobRequired = ( job ) => {

plugins/woocommerce-blocks/package.json

@@ -87,6 +87,7 @@
 		"test:e2e:block-theme": "pnpm run test:e2e block_theme",
 		"test:e2e:classic-theme": "pnpm run test:e2e classic_theme",
 		"test:e2e:block-theme-with-templates": "pnpm run test:e2e block_theme_with_templates",
+		"test:e2e:fake": "echo 'It should trigger Blocks e2e tests in @woocommerce/plugin-woocommerce'",
 		"test:e2e:jest": "pnpm run wp-env:config && cross-env JEST_PUPPETEER_CONFIG=tests/e2e-jest/config/jest-puppeteer.config.js NODE_CONFIG_DIR=tests/e2e-jest/config wp-scripts test-e2e --config tests/e2e-jest/config/jest.config.js",
 		"test:e2e:jest:dev": "pnpm run wp-env:config && cross-env JEST_PUPPETEER_CONFIG=tests/e2e-jest/config/jest-puppeteer.config-dev.js NODE_CONFIG_DIR=tests/e2e-jest/config wp-scripts test-e2e --config tests/e2e-jest/config/jest.config.js",
 		"test:e2e:jest:dev-watch": "pnpm run wp-env:config && cross-env JEST_PUPPETEER_CONFIG=tests/e2e-jest/config/jest-puppeteer.config-dev.js NODE_CONFIG_DIR=tests/e2e-jest/config wp-scripts test-e2e --config tests/e2e-jest/config/jest.config.js --watch",
@@ -370,6 +371,17 @@
 						"pull_request",
 						"push"
 					]
+				},
+				{
+					"name": "Blocks e2e - trigger for Blocks e2e tests in @woocommerce/plugin-woocommerce",
+					"testType": "e2e",
+					"command": "test:e2e:fake",
+					"changes": [
+						"tests/e2e/**"
+					],
+					"events": [
+						"pull_request"
+					]
 				}
 			]
 		}

plugins/woocommerce-blocks/tests/e2e/flaky-tests-reporter.ts

@@ -12,7 +12,13 @@
 import fs from 'fs';
 import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter';
 
+/**
+ * Internal dependencies
+ */
+import pwConfig from './playwright.config';
+
 type FormattedTestResult = Omit< TestResult, 'steps' >;
+const flakyTestsDir = `${ pwConfig.outputDir }/flaky-tests`;
 
 // Remove "steps" to prevent stringify circular structure.
 function formatTestResult( testResult: TestResult ): FormattedTestResult {
@@ -26,7 +32,9 @@ class FlakyTestsReporter implements Reporter {
 
 	onBegin() {
 		try {
-			fs.mkdirSync( 'flaky-tests' );
+			fs.mkdirSync( flakyTestsDir, {
+				recursive: true,
+			} );
 		} catch ( err ) {
 			if (
 				err instanceof Error &&
@@ -56,7 +64,7 @@ class FlakyTestsReporter implements Reporter {
 			case 'flaky': {
 				const safeFileName = testTitle.replace( /[^a-z0-9]/gi, '_' );
 				fs.writeFileSync(
-					`flaky-tests/${ safeFileName }.json`,
+					`${ flakyTestsDir }/${ safeFileName }.json`,
 					JSON.stringify( {
 						version: 1,
 						runner: '@playwright/test',

plugins/woocommerce-blocks/tests/e2e/playwright.config.ts

@@ -10,7 +10,7 @@ const { CI, DEFAULT_TIMEOUT_OVERRIDE } = process.env;
 const config: PlaywrightTestConfig = {
 	maxFailures: 0,
 	timeout: parseInt( DEFAULT_TIMEOUT_OVERRIDE || '', 10 ) || 100_000, // Defaults to 100s.
-	outputDir: './artifacts/test-results',
+	outputDir: `${ __dirname }/artifacts/test-results`,
 	globalSetup: fileURLToPath(
 		new URL( 'global-setup.ts', 'file:' + __filename ).href
 	),

plugins/woocommerce/changelog/ci-merge-blocks-e2e

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Include blocks e2e in ci.yml

plugins/woocommerce/package.json

@@ -27,6 +27,7 @@
 		"env:performance-init": "./tests/performance/bin/init-sample-products.sh",
 		"env:restart": "pnpm wp-env destroy && pnpm wp-env start --update",
 		"env:start": "pnpm wp-env start",
+		"env:start:blocks": "pnpm --filter='@woocommerce/block-library' env:start && pnpm playwright install chromium --with-deps",
 		"env:stop": "pnpm wp-env stop",
 		"env:test": "pnpm env:dev && pnpm playwright install chromium",
 		"env:perf:install-k6": "curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1",
@@ -54,6 +55,7 @@
 		"test:api-pw": "USE_WP_ENV=1 pnpm playwright test --config=tests/api-core-tests/playwright.config.js",
 		"test:e2e-pw": "pnpm playwright test --config=tests/e2e-pw/envs/default/playwright.config.js",
 		"test:e2e": "pnpm test:e2e:with-env default",
+		"test:e2e:blocks": "pnpm --filter='@woocommerce/block-library' test:e2e",
 		"test:e2e:with-env": "bash ./tests/e2e-pw/run-tests-with-env.sh",
 		"test:perf": "./k6 run ./tests/performance/tests/gh-action-pr-requests.js",
 		"test:env:start": "pnpm env:test",
@@ -513,6 +515,40 @@
 						"pull_request",
 						"push"
 					]
+				},
+				{
+					"name": "Blocks e2e tests",
+					"testType": "e2e",
+					"command": "test:e2e:blocks",
+					"shardingArguments": [
+						"--shard=1/10",
+						"--shard=2/10",
+						"--shard=3/10",
+						"--shard=4/10",
+						"--shard=5/10",
+						"--shard=6/10",
+						"--shard=7/10",
+						"--shard=8/10",
+						"--shard=9/10",
+						"--shard=10/10"
+					],
+					"changes": [
+						"src/Blocks/**",
+						"templates/**",
+						"patterns/**"
+					],
+					"testEnv": {
+						"start": "env:start:blocks"
+					},
+					"events": [
+						"pull_request",
+						"push",
+						"release-checks"
+					],
+					"report": {
+						"resultsBlobName": "blocks-e2e-report",
+						"resultsPath": "../woocommerce-blocks/tests/e2e/artifacts/test-results"
+					}
 				}
 			]
 		}