woocommerce/.github/workflows/ci.yml

425 lines
17 KiB
YAML

name: 'CI'
on:
pull_request:
push:
branches:
- 'trunk'
- 'release/*'
release:
types: [ published, edited ]
workflow_call:
inputs:
trigger:
description: 'Type of run to trigger. E.g. daily-e2e, release-checks, etc.'
required: true
default: 'default'
type: string
concurrency:
# Cancel concurrent jobs but not for push event. For push use the run_id to have a unique group.
group: ci-${{ github.event_name == 'push' && github.run_id || github.event_name }}-${{ github.ref }}-${{ inputs.trigger }}
cancel-in-progress: true
env:
FORCE_COLOR: 1
jobs:
project-jobs:
# Since this is a monorepo, not every pull request or change is going to impact every project.
# Instead of running CI tasks on all projects indiscriminately, we use a command to detect
# which projects have changed and what kind of change occurred. This lets us build the
# matrices that we can use to run CI tasks only on the projects that need them.
name: 'Build Project Jobs'
runs-on: 'ubuntu-20.04'
outputs:
lint-jobs: ${{ steps.project-jobs.outputs.lint-jobs }}
test-jobs: ${{ steps.project-jobs.outputs.test-jobs }}
report-jobs: ${{ steps.project-jobs.outputs.report-jobs }}
steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
with:
# If 'base_ref' is available, the 'Build Matrix' step requires non-shallow git-history to identify changed files.
fetch-depth: ${{ ( ( github.base_ref && '0' ) || '1' ) }}
- name: 'Setup PNPM'
uses: 'pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d'
- uses: actions/github-script@v7
name: 'Build Matrix'
id: 'project-jobs'
with:
script: |
// Intended behaviour of the jobs generation:
// - PRs: run CI jobs aiming PRs and filter out jobs based on the content changes
// - Pushes: run CI jobs aiming pushes without filtering based on the content changes
// github.base_ref is only available for pull_request events
let baseRef = ${{ toJson( github.base_ref ) }};
if ( baseRef ) {
baseRef = `--base-ref origin/${ baseRef }`;
}
let githubEvent = ${{ toJson( github.event_name ) }};
const refType = ${{ toJson( github.ref_type ) }};
const refName = ${{ toJson( github.ref_name ) }};
if ( refType === 'tag' && refName !== 'nightly' ) {
githubEvent = 'release-checks';
}
if ( refType === 'tag' && refName === 'nightly' ) {
githubEvent = 'nightly-checks';
}
let trigger = ${{ toJson( inputs.trigger ) }};
if ( trigger ) {
githubEvent = trigger;
}
// `pre-release` should trigger `release-checks`, but without a 'tag' ref.
// This will run all release-checks against the branch the workflow targeted, instead of a release artifact.
if ( trigger === 'pre-release' ) {
githubEvent = 'release-checks';
}
const child_process = require( 'node:child_process' );
child_process.execSync( `pnpm utils ci-jobs ${ baseRef } --event ${ githubEvent }` );
project-lint-jobs:
name: "Lint - ${{ matrix.projectName }} ${{ matrix.optional && ' (optional)' || ''}}"
runs-on: 'ubuntu-20.04'
needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.lint-jobs != '[]' && github.event_name == 'pull_request' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON( needs.project-jobs.outputs.lint-jobs ) }}
steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
with:
# the WooCommerce plugin package uses phpcs-changed for linting, which requires non-shallow git-history.
fetch-depth: ${{ ( ( matrix.projectName == '@woocommerce/plugin-woocommerce' && '0' ) || '1' ) }}
- uses: './.github/actions/setup-woocommerce-monorepo'
name: 'Setup Monorepo'
id: 'setup-monorepo'
with:
install: '${{ matrix.projectName }}...'
pull-package-deps: '${{ matrix.projectName }}'
- name: 'Lint'
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
project-test-jobs:
name: "${{ matrix.name }}"
runs-on: 'ubuntu-20.04'
needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.test-jobs != '[]' }}
env: ${{ matrix.testEnv.envVars }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON( needs.project-jobs.outputs.test-jobs ) }}
steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
- uses: './.github/actions/setup-woocommerce-monorepo'
name: 'Install Monorepo'
id: 'install-monorepo'
with:
install: '${{ matrix.projectName }}...'
build: ${{ ( github.ref_type == 'tag' && 'false' ) || matrix.projectName }}
build-type: ${{ ( matrix.testType == 'unit:php' && 'backend' ) || 'full' }}
pull-playwright-cache: ${{ matrix.testEnv.shouldCreate && matrix.testType == 'e2e' }}
pull-package-deps: '${{ matrix.projectName }}'
- name: 'Update wp-env config'
if: ${{ github.ref_type == 'tag' }}
env:
RELEASE_TAG: ${{ github.ref_name }}
ARTIFACT_NAME: ${{ github.ref_name == 'nightly' && 'woocommerce-trunk-nightly.zip' || 'woocommerce.zip' }}
# band-aid to get the path to wp-env.json for blocks e2e tests, until they're migrated to plugins/woocommerce
WP_ENV_CONFIG_PATH: ${{ github.workspace }}/${{ matrix.testEnv.start == 'env:start:blocks' && 'plugins/woocommerce-blocks' || matrix.projectPath }}
run: node .github/workflows/scripts/override-wp-env-plugins.js
- name: 'Start Test Environment'
id: 'prepare-test-environment'
if: ${{ matrix.testEnv.shouldCreate }}
env: ${{ matrix.testEnv.envVars }}
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
- name: 'Determine BuildKite Analytics Message'
env:
HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
PR_TITLE: ${{ github.event.pull_request.title }}
run: |
if [[ "${{ github.event_name }}" == "push" ]]; then
MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
MESSAGE="$PR_TITLE"
else
MESSAGE="${{ github.event_name }}"
fi
echo "BUILDKITE_ANALYTICS_MESSAGE=$MESSAGE" >> "$GITHUB_OUTPUT"
shell: bash
- name: 'Resolve artifacts path'
if: ${{ always() && matrix.report.resultsPath != '' }}
# Blocks e2e use a relative path which is not supported by actions/upload-artifact@v4
# https://github.com/actions/upload-artifact/issues/176
env:
ARTIFACTS_PATH: '${{ matrix.projectPath }}/${{ matrix.report.resultsPath }}'
run: |
# first runs will probably not have the directory, so we need to create it so that realpath doesn't fail
mkdir -p $ARTIFACTS_PATH
echo "ARTIFACTS_PATH=$(realpath $ARTIFACTS_PATH)" >> $GITHUB_ENV
- name: 'Download Playwright last run info'
id: 'download-last-run-info'
if: ${{ always() && matrix.report.resultsPath != '' && matrix.testType == 'e2e' }}
uses: actions/download-artifact@v4
with:
pattern: 'last-run__${{ strategy.job-index }}'
- name: 'Run tests (${{ matrix.testType }})'
env:
E2E_ENV_KEY: ${{ secrets.E2E_ENV_KEY }}
BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }}
CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} # required by Metrics tests
LAST_FAILED_RUN: ${{ vars.LAST_FAILED_RUN }}
run: |
lastRunFile="${{ steps.download-last-run-info.outputs.download-path }}/last-run__${{ strategy.job-index }}/.last-run.json"
lastRunFileDest="$ARTIFACTS_PATH/.last-run.json"
if [ -f "$lastRunFile" ]; then
echo "Found last run info file: \"$lastRunFile\""
echo "Moving to destination: \"$lastRunFileDest\""
mkdir -p "$ARTIFACTS_PATH"
mv "$lastRunFile" "$lastRunFileDest"
else
echo "No last run info file found. Searched for: \"$lastRunFile\""
fi
lastRunFlag=""
if [ -f "$lastRunFileDest" ]; then
# Playwright last run info is available, parse the file and check if there are failed tests
cat "$lastRunFileDest"
failedTests=$(jq '.failedTests | length' "$lastRunFileDest")
# Only if there are failed tests, we want to use the --last-failed flag.
# The run will fail if we're using the flag and there are no failed tests.
if [ "$failedTests" -gt 0 ]; then
if [ "$LAST_FAILED_RUN" == "1" ]; then
echo "Found failed tests, running only failed tests"
# Add shard 1/1 to override the default shard value. No tests will run for shards > 1.
# The clean way would be to replace the shard flag from the command, but this also works.
lastRunFlag="--last-failed --shard=1/1"
else
echo "Found failed tests, but LAST_FAILED_RUN is switched off. Running all tests."
fi
else
echo "No failed tests found, running all tests"
fi
fi
# Finally, run the tests
pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }} $lastRunFlag
- name: 'Upload Playwright last run info'
# always upload the last run info, even if the test run passed
if: ${{ always() && matrix.report.resultsPath != '' }}
uses: actions/upload-artifact@v4
with:
name: 'last-run__${{ strategy.job-index }}'
path: '${{ env.ARTIFACTS_PATH }}/.last-run.json'
if-no-files-found: ignore
overwrite: true
- name: 'Upload artifacts'
if: ${{ always() && matrix.report.resultsPath != '' }}
uses: actions/upload-artifact@v4
with:
name: '${{ matrix.report.resultsBlobName }}__${{ strategy.job-index }}'
path: ${{ env.ARTIFACTS_PATH }}
- name: 'Upload flaky test reports'
uses: actions/upload-artifact@v4
with:
name: flaky-tests-${{ strategy.job-index }}
path: ${{ env.ARTIFACTS_PATH }}/flaky-tests
if-no-files-found: ignore
evaluate-project-jobs:
# In order to add a required status check we need a consistent job that we can grab onto.
# Since we are dynamically generating a matrix for the project jobs, however, we can't
# rely on any specific job being present. We can get around this limitation by
# using a job that runs after all the others and either passes or fails based
# on the results of the other jobs in the workflow.
name: 'Evaluate Project Job Statuses'
runs-on: 'ubuntu-20.04'
needs:
[
'project-jobs',
'project-lint-jobs',
'project-test-jobs',
]
if: ${{ !cancelled() && github.event_name == 'pull_request' }}
steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
- name: 'Evaluation'
env:
REPOSITORY: ${{ github.repository }}
RUN_ID: ${{ github.run_id }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Check if project-jobs was successful. Fail for any other status, including skipped.
result="${{ needs.project-jobs.result }}"
if [[ $result != "success" ]]; then
echo "Generating CI jobs was not successful."
exit 1
fi
node .github/workflows/scripts/evaluate-jobs-conclusions.js
alert-on-failure:
name: 'Report results on Slack'
runs-on: 'ubuntu-20.04'
needs:
[
'project-jobs',
'project-lint-jobs',
'project-test-jobs',
]
if: ${{ !cancelled() && github.event_name != 'pull_request' && github.repository == 'woocommerce/woocommerce' }}
steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
- name: 'Setup PNPM'
uses: 'pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d'
- name: 'Send messages for failed jobs'
env:
SLACK_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }}
SLACK_CHANNEL: ${{ secrets.TEST_REPORTS_SLACK_CHANNEL }}
HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
INPUT_TRIGGER: ${{ inputs.trigger }}
RUN_TYPE: ${{ github.ref_type == 'tag' && (github.ref_name == 'nightly' && 'nightly-checks' || 'release-checks') || '' }}
run: |
COMMIT_MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
if [[ -n "${INPUT_TRIGGER}" ]]; then
CHECKS_TYPE="${INPUT_TRIGGER}"
else
CHECKS_TYPE="${RUN_TYPE}"
fi
pnpm utils slack-test-report -c "${{ needs.project-jobs.result }}" -r "$CHECKS_TYPE Build jobs matrix" -m "$COMMIT_MESSAGE"
pnpm utils slack-test-report -c "${{ needs.project-lint-jobs.result }}" -r "$CHECKS_TYPE Linting" -m "$COMMIT_MESSAGE"
pnpm utils slack-test-report -c "${{ needs.project-test-jobs.result }}" -r "$CHECKS_TYPE Tests" -m "$COMMIT_MESSAGE"
test-reports:
name: 'Test reports - ${{ matrix.report }}'
needs:
[
'project-jobs',
'project-test-jobs',
]
if: ${{ !cancelled() && needs.project-jobs.outputs.report-jobs != '[]' && github.repository == 'woocommerce/woocommerce' }}
strategy:
fail-fast: false
matrix:
report: ${{ fromJSON( needs.project-jobs.outputs.report-jobs ) }}
runs-on: ubuntu-latest
env:
ARTIFACT_NAME: ${{ matrix.report }}-attempt-${{ github.run_attempt }}
steps:
- uses: actions/checkout@v4
- name: 'Merge artifacts'
id: merge-artifacts
uses: actions/upload-artifact/merge@v4
continue-on-error: true
with:
name: ${{ env.ARTIFACT_NAME }}
pattern: ${{ matrix.report }}__*
delete-merged: true
- name: 'Publish report to dashboard'
if: ${{ !! steps.merge-artifacts.outputs.artifact-id }}
env:
GH_TOKEN: ${{ secrets.REPORTS_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
REPORT_NAME: ${{ matrix.report }}
HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
PR_TITLE: ${{ github.event.pull_request.title }}
EVENT_NAME: ${{ inputs.trigger == '' && github.event_name || inputs.trigger }}
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
REPORT_TITLE="$PR_TITLE"
REF_NAME="$GITHUB_HEAD_REF"
elif [[ "${{ github.event_name }}" == "push" ]]; then
REPORT_TITLE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
REF_NAME="$GITHUB_REF_NAME"
else
REPORT_TITLE="$EVENT_NAME"
REF_NAME="$GITHUB_REF_NAME"
fi
gh workflow run report.yml \
-f artifact="$ARTIFACT_NAME" \
-f run_id="$GITHUB_RUN_ID" \
-f run_attempt="$GITHUB_RUN_ATTEMPT" \
-f event="$EVENT_NAME" \
-f pr_number="$PR_NUMBER" \
-f ref_name="$REF_NAME" \
-f commit_sha="$GITHUB_SHA" \
-f repository="$GITHUB_REPOSITORY" \
-f suite="$REPORT_NAME" \
-f report_title="$REPORT_TITLE" \
--repo woocommerce/woocommerce-test-reports
report-flaky-tests:
name: 'Create issues for flaky tests'
if: ${{ !cancelled() && github.repository == 'woocommerce/woocommerce' && needs.project-jobs.outputs.test-jobs != '[]' }}
needs:
[
'project-jobs',
'project-test-jobs',
]
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
- uses: 'actions/download-artifact@v4'
name: 'Download artifacts'
with:
pattern: flaky-tests*
path: flaky-tests
merge-multiple: true
- name: 'Merge flaky tests reports'
run: |
downloadPath='${{ steps.download-artifact.outputs.download-path || './flaky-tests' }}'
# make dir so that next step doesn't fail if it doesn't exist
mkdir -p $downloadPath
# any output means there are reports
echo "FLAKY_REPORTS=$(ls -A $downloadPath | head -1)" >> $GITHUB_ENV
- name: 'Report flaky tests'
if: ${{ !!env.FLAKY_REPORTS }}
uses: './.github/actions/report-flaky-tests'
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
label: 'metric: flaky e2e test'