From adbdcceb28b0305ca1a1056f7329a0c614eae6dc Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Tue, 6 Aug 2024 22:50:25 +0100 Subject: [PATCH] [testing workflows] Only re-run failed e2e tests (#50267) * Use Playwright's last info run * Add changelog * Fix workflow syntax * Fix workflow syntax * Try with continue-on-error * Try with merge-multiple * Try with pattern instead of name * Print the run command * Print the run command * Use LAST_RUN_FLAG * Update outputDir * Update upload paths and parse the file content * Don't try to download lat run info if the test type is not e2e * Rename the downloaded file * Rename the downloaded file * Rename the downloaded file * Full path to the downloaded file * Check download path content * Check download path content * Check the last-run file in the right path * Fail different tests on different run attempts * Fix artifact name * Force test failures * Cleanup console logs * Cleanup console logs * Cleanup console logs * Log if last run info exists but no tests are found * Revert test changes --- .github/workflows/ci.yml | 72 +++++++++++++++---- .../ci-only-run-last-failed-e2e-tests | 4 ++ .../tests/e2e-pw/playwright.config.js | 2 +- 3 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 plugins/woocommerce/changelog/ci-only-run-last-failed-e2e-tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 062ec146df9..08cff063649 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,30 +150,21 @@ jobs: env: ${{ matrix.testEnv.envVars }} run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}' - - name: 'Get commit message' - id: 'get_commit_message' + - name: 'Determine BuildKite Analytics Message' env: HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} PR_TITLE: ${{ github.event.pull_request.title }} run: | if [[ "${{ github.event_name }}" == "push" ]]; then - COMMIT_MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1` + MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1` elif [[ "${{ github.event_name }}" == "pull_request" ]]; then - COMMIT_MESSAGE="$PR_TITLE" + MESSAGE="$PR_TITLE" else - COMMIT_MESSAGE="${{ github.event_name }}" + MESSAGE="${{ github.event_name }}" fi - echo "COMMIT_MESSAGE=$COMMIT_MESSAGE" >> "$GITHUB_OUTPUT" + echo "BUILDKITE_ANALYTICS_MESSAGE=$MESSAGE" >> "$GITHUB_OUTPUT" shell: bash - - name: 'Run tests (${{ matrix.testType }})' - env: - E2E_ENV_KEY: ${{ secrets.E2E_ENV_KEY }} - BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }} - BUILDKITE_ANALYTICS_MESSAGE: ${{ steps.get_commit_message.outputs.COMMIT_MESSAGE }} - CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} # required by Metrics tests - run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}' - - name: 'Resolve artifacts path' if: ${{ always() && matrix.report.resultsPath != '' }} # Blocks e2e use a relative path which is not supported by actions/upload-artifact@v4 @@ -182,6 +173,59 @@ jobs: ARTIFACTS_PATH: '${{ matrix.projectPath }}/${{ matrix.report.resultsPath }}' run: echo "ARTIFACTS_PATH=$(realpath $ARTIFACTS_PATH)" >> $GITHUB_ENV + - name: 'Download Playwright last run info' + id: 'download-last-run-info' + if: ${{ always() && matrix.report.resultsPath != '' && matrix.testType == 'e2e' }} + uses: actions/download-artifact@v4 + with: + pattern: 'last-run__${{ strategy.job-index }}' + + - name: 'Run tests (${{ matrix.testType }})' + env: + E2E_ENV_KEY: ${{ secrets.E2E_ENV_KEY }} + BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }} + CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} # required by Metrics tests + run: | + lastRunFile="${{ steps.download-last-run-info.outputs.download-path }}/last-run__${{ strategy.job-index }}/.last-run.json" + lastRunFileDest="$ARTIFACTS_PATH/.last-run.json" + + if [ -f "$lastRunFile" ]; then + echo "Found last run info file: \"$lastRunFile\"" + echo "Moving to destination: \"$lastRunFileDest\"" + mkdir -p "$ARTIFACTS_PATH" + mv "$lastRunFile" "$lastRunFileDest" + else + echo "No last run info file found. Searched for: \"$lastRunFile\"" + fi + + lastRunFlag="" + if [ -f "$lastRunFileDest" ]; then + # Playwright last run info is available, parse the file and check if there are failed tests + failedTests=$(jq '.failedTests | length' "$lastRunFileDest") + + # Only if there are failed tests, we want to use the --last-failed flag. + # The run will fail if we're using the flag and there are no failed tests. + if [ "$failedTests" -gt 0 ]; then + echo "Found failed tests, running only failed tests" + lastRunFlag="--last-failed" + else + echo "No failed tests found, running all tests" + fi + fi + + # Finally, run the tests + pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }} $lastRunFlag + + - name: 'Upload Playwright last run info' + # always upload the last run info, even if the test run passed + if: ${{ always() && matrix.report.resultsPath != '' }} + uses: actions/upload-artifact@v4 + with: + name: 'last-run__${{ strategy.job-index }}' + path: '${{ env.ARTIFACTS_PATH }}/.last-run.json' + if-no-files-found: ignore + overwrite: true + - name: 'Upload artifacts' if: ${{ always() && matrix.report.resultsPath != '' }} uses: actions/upload-artifact@v4 diff --git a/plugins/woocommerce/changelog/ci-only-run-last-failed-e2e-tests b/plugins/woocommerce/changelog/ci-only-run-last-failed-e2e-tests new file mode 100644 index 00000000000..4f73f5cc6c2 --- /dev/null +++ b/plugins/woocommerce/changelog/ci-only-run-last-failed-e2e-tests @@ -0,0 +1,4 @@ +Significance: patch +Type: dev + + diff --git a/plugins/woocommerce/tests/e2e-pw/playwright.config.js b/plugins/woocommerce/tests/e2e-pw/playwright.config.js index 49046e0be84..6416a0dceb1 100644 --- a/plugins/woocommerce/tests/e2e-pw/playwright.config.js +++ b/plugins/woocommerce/tests/e2e-pw/playwright.config.js @@ -67,7 +67,7 @@ const config = { ? Number( DEFAULT_TIMEOUT_OVERRIDE ) : 120 * 1000, expect: { timeout: 20 * 1000 }, - outputDir: `${ testsResultsPath }/results-data`, + outputDir: testsResultsPath, globalSetup: require.resolve( './global-setup' ), globalTeardown: require.resolve( './global-teardown' ), testDir: `${ testsRootPath }/tests`,