[testing workflows] Only re-run failed e2e tests (#50267)
* Use Playwright's last info run * Add changelog * Fix workflow syntax * Fix workflow syntax * Try with continue-on-error * Try with merge-multiple * Try with pattern instead of name * Print the run command * Print the run command * Use LAST_RUN_FLAG * Update outputDir * Update upload paths and parse the file content * Don't try to download lat run info if the test type is not e2e * Rename the downloaded file * Rename the downloaded file * Rename the downloaded file * Full path to the downloaded file * Check download path content * Check download path content * Check the last-run file in the right path * Fail different tests on different run attempts * Fix artifact name * Force test failures * Cleanup console logs * Cleanup console logs * Cleanup console logs * Log if last run info exists but no tests are found * Revert test changes
This commit is contained in:
parent
0a7d23e868
commit
adbdcceb28
|
@ -150,30 +150,21 @@ jobs:
|
||||||
env: ${{ matrix.testEnv.envVars }}
|
env: ${{ matrix.testEnv.envVars }}
|
||||||
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
|
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
|
||||||
|
|
||||||
- name: 'Get commit message'
|
- name: 'Determine BuildKite Analytics Message'
|
||||||
id: 'get_commit_message'
|
|
||||||
env:
|
env:
|
||||||
HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
|
HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
|
||||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||||
run: |
|
run: |
|
||||||
if [[ "${{ github.event_name }}" == "push" ]]; then
|
if [[ "${{ github.event_name }}" == "push" ]]; then
|
||||||
COMMIT_MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
|
MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
|
||||||
elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
|
elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
|
||||||
COMMIT_MESSAGE="$PR_TITLE"
|
MESSAGE="$PR_TITLE"
|
||||||
else
|
else
|
||||||
COMMIT_MESSAGE="${{ github.event_name }}"
|
MESSAGE="${{ github.event_name }}"
|
||||||
fi
|
fi
|
||||||
echo "COMMIT_MESSAGE=$COMMIT_MESSAGE" >> "$GITHUB_OUTPUT"
|
echo "BUILDKITE_ANALYTICS_MESSAGE=$MESSAGE" >> "$GITHUB_OUTPUT"
|
||||||
shell: bash
|
shell: bash
|
||||||
|
|
||||||
- name: 'Run tests (${{ matrix.testType }})'
|
|
||||||
env:
|
|
||||||
E2E_ENV_KEY: ${{ secrets.E2E_ENV_KEY }}
|
|
||||||
BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }}
|
|
||||||
BUILDKITE_ANALYTICS_MESSAGE: ${{ steps.get_commit_message.outputs.COMMIT_MESSAGE }}
|
|
||||||
CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} # required by Metrics tests
|
|
||||||
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
|
|
||||||
|
|
||||||
- name: 'Resolve artifacts path'
|
- name: 'Resolve artifacts path'
|
||||||
if: ${{ always() && matrix.report.resultsPath != '' }}
|
if: ${{ always() && matrix.report.resultsPath != '' }}
|
||||||
# Blocks e2e use a relative path which is not supported by actions/upload-artifact@v4
|
# Blocks e2e use a relative path which is not supported by actions/upload-artifact@v4
|
||||||
|
@ -182,6 +173,59 @@ jobs:
|
||||||
ARTIFACTS_PATH: '${{ matrix.projectPath }}/${{ matrix.report.resultsPath }}'
|
ARTIFACTS_PATH: '${{ matrix.projectPath }}/${{ matrix.report.resultsPath }}'
|
||||||
run: echo "ARTIFACTS_PATH=$(realpath $ARTIFACTS_PATH)" >> $GITHUB_ENV
|
run: echo "ARTIFACTS_PATH=$(realpath $ARTIFACTS_PATH)" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: 'Download Playwright last run info'
|
||||||
|
id: 'download-last-run-info'
|
||||||
|
if: ${{ always() && matrix.report.resultsPath != '' && matrix.testType == 'e2e' }}
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
pattern: 'last-run__${{ strategy.job-index }}'
|
||||||
|
|
||||||
|
- name: 'Run tests (${{ matrix.testType }})'
|
||||||
|
env:
|
||||||
|
E2E_ENV_KEY: ${{ secrets.E2E_ENV_KEY }}
|
||||||
|
BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }}
|
||||||
|
CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} # required by Metrics tests
|
||||||
|
run: |
|
||||||
|
lastRunFile="${{ steps.download-last-run-info.outputs.download-path }}/last-run__${{ strategy.job-index }}/.last-run.json"
|
||||||
|
lastRunFileDest="$ARTIFACTS_PATH/.last-run.json"
|
||||||
|
|
||||||
|
if [ -f "$lastRunFile" ]; then
|
||||||
|
echo "Found last run info file: \"$lastRunFile\""
|
||||||
|
echo "Moving to destination: \"$lastRunFileDest\""
|
||||||
|
mkdir -p "$ARTIFACTS_PATH"
|
||||||
|
mv "$lastRunFile" "$lastRunFileDest"
|
||||||
|
else
|
||||||
|
echo "No last run info file found. Searched for: \"$lastRunFile\""
|
||||||
|
fi
|
||||||
|
|
||||||
|
lastRunFlag=""
|
||||||
|
if [ -f "$lastRunFileDest" ]; then
|
||||||
|
# Playwright last run info is available, parse the file and check if there are failed tests
|
||||||
|
failedTests=$(jq '.failedTests | length' "$lastRunFileDest")
|
||||||
|
|
||||||
|
# Only if there are failed tests, we want to use the --last-failed flag.
|
||||||
|
# The run will fail if we're using the flag and there are no failed tests.
|
||||||
|
if [ "$failedTests" -gt 0 ]; then
|
||||||
|
echo "Found failed tests, running only failed tests"
|
||||||
|
lastRunFlag="--last-failed"
|
||||||
|
else
|
||||||
|
echo "No failed tests found, running all tests"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Finally, run the tests
|
||||||
|
pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }} $lastRunFlag
|
||||||
|
|
||||||
|
- name: 'Upload Playwright last run info'
|
||||||
|
# always upload the last run info, even if the test run passed
|
||||||
|
if: ${{ always() && matrix.report.resultsPath != '' }}
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: 'last-run__${{ strategy.job-index }}'
|
||||||
|
path: '${{ env.ARTIFACTS_PATH }}/.last-run.json'
|
||||||
|
if-no-files-found: ignore
|
||||||
|
overwrite: true
|
||||||
|
|
||||||
- name: 'Upload artifacts'
|
- name: 'Upload artifacts'
|
||||||
if: ${{ always() && matrix.report.resultsPath != '' }}
|
if: ${{ always() && matrix.report.resultsPath != '' }}
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v4
|
||||||
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
Significance: patch
|
||||||
|
Type: dev
|
||||||
|
|
||||||
|
|
|
@ -67,7 +67,7 @@ const config = {
|
||||||
? Number( DEFAULT_TIMEOUT_OVERRIDE )
|
? Number( DEFAULT_TIMEOUT_OVERRIDE )
|
||||||
: 120 * 1000,
|
: 120 * 1000,
|
||||||
expect: { timeout: 20 * 1000 },
|
expect: { timeout: 20 * 1000 },
|
||||||
outputDir: `${ testsResultsPath }/results-data`,
|
outputDir: testsResultsPath,
|
||||||
globalSetup: require.resolve( './global-setup' ),
|
globalSetup: require.resolve( './global-setup' ),
|
||||||
globalTeardown: require.resolve( './global-teardown' ),
|
globalTeardown: require.resolve( './global-teardown' ),
|
||||||
testDir: `${ testsRootPath }/tests`,
|
testDir: `${ testsRootPath }/tests`,
|
||||||
|
|
Loading…
Reference in New Issue