[test reporting] Include list of failed tests in Slack notifications (#46360)

.github/workflows/ci.yml

@@ -125,8 +125,8 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
-          name: all-blob-reports-${{ matrix.shardNumber }}
-          path: ${{ matrix.projectPath }}/tests/e2e-pw/test-results/allure-results
+          name: all-blob-e2e-reports-${{ matrix.shardNumber }}
+          path: ${{ matrix.projectPath }}/tests/e2e-pw/test-results
           retention-days: 1
           compression-level: 9
           
@@ -162,7 +162,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
-          name: all-blob-reports-${{ matrix.shardNumber }}
+          name: all-blob-api-reports-${{ matrix.shardNumber }}
           path: ${{ matrix.projectPath }}/tests/api-core-tests/test-results/allure-results
           retention-days: 1
           compression-level: 9
@@ -258,8 +258,8 @@ jobs:
       - name: 'Download blob reports from artifacts'
         uses: actions/download-artifact@v4
         with:
-          path: ./out/allure-results
-          pattern: all-blob-reports-*
+          path: ./out
+          pattern: all-blob-e2e-reports-*
           run-id: project-e2e-test-jobs
           merge-multiple: true
 
@@ -306,6 +306,8 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           slack_token: ${{ secrets.E2E_SLACK_TOKEN }}
           slack_channel: ${{ secrets.E2E_TRUNK_SLACK_CHANNEL }}
+          playwright_report_path: ./out/test-results-*.json
+          playwright_output_dir: ./out/results-data
 
   api-test-reports:
     name: 'Report API tests results'
@@ -325,7 +327,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           path: ./out/allure-results
-          pattern: all-blob-reports-*
+          pattern: all-blob-api-reports-*
           run-id: project-api-test-jobs
           merge-multiple: true
 

plugins/woocommerce/changelog/ci-add-failed-tests-in-slack-notification

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Include list of failed tests in Slack notifications

plugins/woocommerce/tests/e2e-pw/.gitignore

@@ -0,0 +1,3 @@
+/.state
+/test-results
+

plugins/woocommerce/tests/e2e-pw/playwright.config.js

@@ -11,12 +11,43 @@ const {
 	REPEAT_EACH,
 } = process.env;
 
+const reporter = [
+	[ 'list' ],
+	[
+		'allure-playwright',
+		{
+			outputFolder:
+				ALLURE_RESULTS_DIR ??
+				'./tests/e2e-pw/test-results/allure-results',
+			detail: true,
+			suiteTitle: true,
+		},
+	],
+	[
+		'json',
+		{ outputFile: `./test-results/test-results-${ Date.now() }.json` },
+	],
+];
+
+if ( process.env.CI ) {
+	reporter.push( [ 'github' ] );
+} else {
+	reporter.push( [
+		'html',
+		{
+			outputFolder:
+				PLAYWRIGHT_HTML_REPORT ?? './test-results/playwright-report',
+			open: 'on-failure',
+		},
+	] );
+}
+
 const config = {
 	timeout: DEFAULT_TIMEOUT_OVERRIDE
 		? Number( DEFAULT_TIMEOUT_OVERRIDE )
 		: 120 * 1000,
 	expect: { timeout: 20 * 1000 },
-	outputDir: './test-results/report',
+	outputDir: './test-results/results-data',
 	globalSetup: require.resolve( './global-setup' ),
 	globalTeardown: require.resolve( './global-teardown' ),
 	testDir: 'tests',
@@ -24,35 +55,12 @@ const config = {
 	repeatEach: REPEAT_EACH ? Number( REPEAT_EACH ) : 1,
 	workers: 1,
 	reportSlowTests: { max: 5, threshold: 30 * 1000 }, // 30 seconds threshold
-	reporter: [
-		[ 'list' ],
-		[
-			'html',
-			{
-				outputFolder:
-					PLAYWRIGHT_HTML_REPORT ??
-					'./test-results/playwright-report',
-				open: CI ? 'never' : 'always',
-			},
-		],
-		[
-			'allure-playwright',
-			{
-				outputFolder:
-					ALLURE_RESULTS_DIR ??
-					'./tests/e2e-pw/test-results/allure-results',
-				detail: true,
-				suiteTitle: true,
-			},
-		],
-		[ 'json', { outputFile: './test-results/test-results.json' } ],
-		[ 'github' ],
-	],
+	reporter,
 	maxFailures: E2E_MAX_FAILURES ? Number( E2E_MAX_FAILURES ) : 0,
 	use: {
 		baseURL: BASE_URL ?? 'http://localhost:8086',
 		screenshot: { mode: 'only-on-failure', fullPage: true },
-		stateDir: 'tests/e2e-pw/test-results/storage/',
+		stateDir: 'tests/e2e-pw/.state/',
 		trace: 'retain-on-failure',
 		video: 'retain-on-failure',
 		viewport: { width: 1280, height: 720 },