[testing workflows] Only re-run failed e2e tests (#50267)

* Use Playwright's last info run

* Add changelog

* Fix workflow syntax

* Fix workflow syntax

* Try with continue-on-error

* Try with merge-multiple

* Try with pattern instead of name

* Print the run command

* Print the run command

* Use LAST_RUN_FLAG

* Update outputDir

* Update upload paths and parse the file content

* Don't try to download lat run info if the test type is not e2e

* Rename the downloaded file

* Rename the downloaded file

* Rename the downloaded file

* Full path to the downloaded file

* Check download path content

* Check download path content

* Check the last-run file in the right path

* Fail different tests on different run attempts

* Fix artifact name

* Force test failures

* Cleanup console logs

* Cleanup console logs

* Cleanup console logs

* Log if last run info exists but no tests are found

* Revert test changes

.github/workflows/ci.yml

@@ -150,30 +150,21 @@ jobs:
         env: ${{ matrix.testEnv.envVars }}
         run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
 
-      - name: 'Get commit message'
-        id: 'get_commit_message'
+      - name: 'Determine BuildKite Analytics Message'
         env:
           HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
           PR_TITLE: ${{ github.event.pull_request.title }}
         run: |
           if [[ "${{ github.event_name }}" == "push" ]]; then
-            COMMIT_MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
+            MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
           elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            COMMIT_MESSAGE="$PR_TITLE"
+            MESSAGE="$PR_TITLE"
           else
-            COMMIT_MESSAGE="${{ github.event_name }}"
+            MESSAGE="${{ github.event_name }}"
           fi
-          echo "COMMIT_MESSAGE=$COMMIT_MESSAGE" >> "$GITHUB_OUTPUT"
+          echo "BUILDKITE_ANALYTICS_MESSAGE=$MESSAGE" >> "$GITHUB_OUTPUT"
         shell: bash
 
-      - name: 'Run tests (${{ matrix.testType }})'
-        env:
-          E2E_ENV_KEY: ${{ secrets.E2E_ENV_KEY }}
-          BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }}
-          BUILDKITE_ANALYTICS_MESSAGE: ${{ steps.get_commit_message.outputs.COMMIT_MESSAGE }}
-          CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} # required by Metrics tests
-        run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
-
       - name: 'Resolve artifacts path'
         if: ${{ always() && matrix.report.resultsPath != '' }}
         # Blocks e2e use a relative path which is not supported by actions/upload-artifact@v4
@@ -182,6 +173,59 @@ jobs:
           ARTIFACTS_PATH: '${{ matrix.projectPath }}/${{ matrix.report.resultsPath }}'
         run: echo "ARTIFACTS_PATH=$(realpath $ARTIFACTS_PATH)" >> $GITHUB_ENV
 
+      - name: 'Download Playwright last run info'
+        id: 'download-last-run-info'
+        if: ${{ always() && matrix.report.resultsPath != '' && matrix.testType == 'e2e' }}
+        uses: actions/download-artifact@v4
+        with:
+          pattern: 'last-run__${{ strategy.job-index }}'
+
+      - name: 'Run tests (${{ matrix.testType }})'
+        env:
+          E2E_ENV_KEY: ${{ secrets.E2E_ENV_KEY }}
+          BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }}
+          CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} # required by Metrics tests
+        run: |
+          lastRunFile="${{ steps.download-last-run-info.outputs.download-path }}/last-run__${{ strategy.job-index }}/.last-run.json"
+          lastRunFileDest="$ARTIFACTS_PATH/.last-run.json"
+          
+          if [ -f "$lastRunFile" ]; then
+              echo "Found last run info file: \"$lastRunFile\""
+              echo "Moving to destination: \"$lastRunFileDest\""
+              mkdir -p "$ARTIFACTS_PATH"
+              mv "$lastRunFile" "$lastRunFileDest"
+          else
+              echo "No last run info file found. Searched for: \"$lastRunFile\""
+          fi
+          
+          lastRunFlag=""
+          if [ -f "$lastRunFileDest" ]; then
+            # Playwright last run info is available, parse the file and check if there are failed tests
+            failedTests=$(jq '.failedTests | length' "$lastRunFileDest")
+          
+            # Only if there are failed tests, we want to use the --last-failed flag. 
+            # The run will fail if we're using the flag and there are no failed tests.
+            if [ "$failedTests" -gt 0 ]; then
+              echo "Found failed tests, running only failed tests"
+              lastRunFlag="--last-failed"
+            else
+                echo "No failed tests found, running all tests"
+            fi
+          fi
+          
+          # Finally, run the tests
+          pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }} $lastRunFlag
+          
+      - name: 'Upload Playwright last run info'
+        # always upload the last run info, even if the test run passed
+        if: ${{ always() && matrix.report.resultsPath != '' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: 'last-run__${{ strategy.job-index }}'
+          path: '${{ env.ARTIFACTS_PATH }}/.last-run.json'
+          if-no-files-found: ignore
+          overwrite: true
+
       - name: 'Upload artifacts'
         if: ${{ always() && matrix.report.resultsPath != '' }}
         uses: actions/upload-artifact@v4

plugins/woocommerce/changelog/ci-only-run-last-failed-e2e-tests

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+

plugins/woocommerce/tests/e2e-pw/playwright.config.js

@@ -67,7 +67,7 @@ const config = {
 		? Number( DEFAULT_TIMEOUT_OVERRIDE )
 		: 120 * 1000,
 	expect: { timeout: 20 * 1000 },
-	outputDir: `${ testsResultsPath }/results-data`,
+	outputDir: testsResultsPath,
 	globalSetup: require.resolve( './global-setup' ),
 	globalTeardown: require.resolve( './global-teardown' ),
 	testDir: `${ testsRootPath }/tests`,