woocommerce/.github/workflows/ci.yml

name: 'CI'
on:
    pull_request:
    push:
        branches:
            - 'trunk'
            - 'release/*'
    workflow_call:
        inputs:
            trigger:
                description: 'Type of run to trigger. E.g. daily-e2e, release-checks, etc.'
                required: true
                default: 'default'
                type: string

concurrency:
    group: '${{ github.workflow }}-${{ github.ref }}'
    cancel-in-progress: true
    
env: 
  FORCE_COLOR: 1

jobs:
    project-jobs:
        # Since this is a monorepo, not every pull request or change is going to impact every project.
        # Instead of running CI tasks on all projects indiscriminately, we use a command to detect
        # which projects have changed and what kind of change occurred. This lets us build the
        # matrices that we can use to run CI tasks only on the projects that need them.
        name: 'Build Project Jobs'
        runs-on: 'ubuntu-20.04'
        outputs:
            lint-jobs: ${{ steps.project-jobs.outputs.lint-jobs }}
            default-test-jobs: ${{ steps.project-jobs.outputs.default-test-jobs }}
            e2e-test-jobs: ${{ steps.project-jobs.outputs.e2e-test-jobs }}
            api-test-jobs: ${{ steps.project-jobs.outputs.api-test-jobs }}
            performance-test-jobs: ${{ steps.project-jobs.outputs.performance-test-jobs }}
        steps:
            - uses: 'actions/checkout@v4'
              name: 'Checkout'
              with:
                  fetch-depth: 0
            - uses: './.github/actions/setup-woocommerce-monorepo'
              name: 'Setup Monorepo'
              with:
                  php-version: false # We don't want to waste time installing PHP since we aren't using it in this job.
            - uses: actions/github-script@v7
              name: 'Build Matrix'
              id: 'project-jobs'
              with:
                  script: |
                      let baseRef = ${{ toJson( github.base_ref ) }};
                      if ( baseRef ) {
                        baseRef = `--base-ref origin/${ baseRef }`;
                      }

                      let githubEvent = ${{ toJson( github.event_name ) }};
                      let trigger = ${{ toJson( inputs.trigger ) }};

                      if ( trigger ) {
                        githubEvent = trigger;
                      }

                      const child_process = require( 'node:child_process' );
                      child_process.execSync( `pnpm utils ci-jobs ${ baseRef } --event ${ githubEvent }` );

    project-lint-jobs:
        name: "Lint - ${{ matrix.projectName }} ${{ matrix.optional && ' (optional)' || ''}}"
        runs-on: 'ubuntu-20.04'
        needs: 'project-jobs'
        if: ${{ needs.project-jobs.outputs.lint-jobs != '[]' && github.event_name == 'pull_request' }}
        strategy:
            fail-fast: false
            matrix:
                include: ${{ fromJSON( needs.project-jobs.outputs.lint-jobs ) }}
        steps:
            - uses: 'actions/checkout@v4'
              name: 'Checkout'
              with:
                  fetch-depth: 0
            - uses: './.github/actions/setup-woocommerce-monorepo'
              name: 'Setup Monorepo'
              id: 'setup-monorepo'
              with:
                  install: '${{ matrix.projectName }}...'
                  build: '${{ matrix.projectName }}'
            - name: 'Lint'
              run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'

    project-default-test-jobs:
        name: "Test - ${{ matrix.projectName }} - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || '' || ''}}"
        runs-on: 'ubuntu-20.04'
        needs: 'project-jobs'
        if: ${{ needs.project-jobs.outputs.default-test-jobs != '[]' }}
        strategy:
            fail-fast: false
            matrix:
                include: ${{ fromJSON( needs.project-jobs.outputs.default-test-jobs ) }}
        steps:
            - uses: 'actions/checkout@v4'
              name: 'Checkout'
            - uses: './.github/actions/setup-woocommerce-monorepo'
              name: 'Setup Monorepo'
              id: 'setup-monorepo'
              with:
                  install: '${{ matrix.projectName }}...'
                  build: '${{ matrix.projectName }}'
            - name: 'Start Test Environment'
              id: 'prepare-test-environment'
              if: ${{ matrix.testEnv.shouldCreate }}
              env: ${{ matrix.testEnv.envVars }}
              run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
            - name: 'Test'
              run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'

    project-e2e-test-jobs:
        name: "E2E - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
        runs-on: 'ubuntu-20.04'
        needs: 'project-jobs'
        if: ${{ needs.project-jobs.outputs.e2e-test-jobs != '[]' }}
        env: ${{ matrix.testEnv.envVars }}
        strategy:
            fail-fast: false
            matrix:
                include: ${{ fromJSON( needs.project-jobs.outputs.e2e-test-jobs ) }}
        steps:
            - uses: 'actions/checkout@v4'
              name: 'Checkout'

            - uses: './.github/actions/setup-woocommerce-monorepo'
              name: 'Setup Monorepo'
              id: 'setup-monorepo'
              with:
                  install: '${{ matrix.projectName }}...'
                  build: '${{ matrix.projectName }}'

            - name: Get commit message
              id: get_commit_message
              env:
                  HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
                  PR_TITLE: ${{ github.event.pull_request.title }}
              run: |
                  if [[ "${{ github.event_name }}" == "push" ]]; then
                    COMMIT_MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
                  elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
                    COMMIT_MESSAGE="$PR_TITLE"
                  else
                    COMMIT_MESSAGE="${{ github.event_name }}"
                  fi
                  echo "COMMIT_MESSAGE=$COMMIT_MESSAGE" >> "$GITHUB_OUTPUT"
              shell: bash

            - name: 'Start Test Environment'
              id: 'prepare-test-environment'
              if: ${{ matrix.testEnv.shouldCreate }}
              run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'

            - name: 'Run tests'
              env:
                  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }}
                  BUILDKITE_ANALYTICS_MESSAGE: ${{ steps.get_commit_message.outputs.COMMIT_MESSAGE }}
              run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'

            - name: 'Upload artifacts'
              if: ${{ always() }}
              uses: actions/upload-artifact@v4
              with:
                  name: all-blob-e2e-reports-${{ strategy.job-index }}
                  path: ${{ matrix.projectPath }}/tests/e2e-pw/test-results
                  retention-days: 1
                  compression-level: 9

    project-api-test-jobs:
        name: "API - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
        runs-on: 'ubuntu-20.04'
        needs: 'project-jobs'
        if: ${{ needs.project-jobs.outputs.api-test-jobs != '[]' }}
        env: ${{ matrix.testEnv.envVars }}
        strategy:
            fail-fast: false
            matrix:
                include: ${{ fromJSON( needs.project-jobs.outputs.api-test-jobs ) }}
        steps:
            - uses: 'actions/checkout@v4'
              name: 'Checkout'

            - uses: './.github/actions/setup-woocommerce-monorepo'
              name: 'Setup Monorepo'
              id: 'setup-monorepo'
              with:
                  install: '${{ matrix.projectName }}...'
                  build: '${{ matrix.projectName }}'

            - name: 'Start Test Environment'
              id: 'prepare-test-environment'
              if: ${{ matrix.testEnv.shouldCreate }}
              run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'

            - name: 'Run tests'
              run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'

            - name: 'Upload artifacts'
              if: ${{ always() }}
              uses: actions/upload-artifact@v4
              with:
                  name: all-blob-api-reports-${{ strategy.job-index }}
                  path: ${{ matrix.projectPath }}/tests/api-core-tests/test-results/allure-results
                  retention-days: 1
                  compression-level: 9

    project-performance-test-jobs:
        name: "Performance - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
        runs-on: 'ubuntu-20.04'
        needs: 'project-jobs'
        if: ${{ needs.project-jobs.outputs.performance-test-jobs != '[]' }}
        strategy:
            fail-fast: false
            matrix:
                include: ${{ fromJSON( needs.project-jobs.outputs.performance-test-jobs ) }}
        env:
            WP_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts

        steps:
            - uses: 'actions/checkout@v4'
              name: 'Checkout'

            - uses: './.github/actions/setup-woocommerce-monorepo'
              name: 'Setup Monorepo'
              id: 'setup-monorepo'
              with:
                  install: '${{ matrix.projectName }}...'
                  build: '${{ matrix.projectName }}'

            - name: 'Start Test Environment'
              id: 'prepare-test-environment'
              if: ${{ matrix.testEnv.shouldCreate }}
              run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'

            - name: 'Run tests'
              env:
                  CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }}
              run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'

            - name: 'Archive metrics results'
              if: ${{ success() && matrix.name == 'Metrics' }}
              uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
              with:
                  name: metrics-results
                  path: ${{ env.WP_ARTIFACTS_PATH }}/*.performance-results*.json

    evaluate-project-jobs:
        # In order to add a required status check we need a consistent job that we can grab onto.
        # Since we are dynamically generating a matrix for the project jobs, however, we can't
        # rely on any specific job being present. We can get around this limitation by
        # using a job that runs after all the others and either passes or fails based
        # on the results of the other jobs in the workflow.
        name: 'Evaluate Project Job Statuses'
        runs-on: 'ubuntu-20.04'
        needs:
            [
                'project-jobs',
                'project-lint-jobs',
                'project-default-test-jobs',
                'project-e2e-test-jobs',
                'project-api-test-jobs',
                'project-performance-test-jobs',
            ]
        if: ${{ always() && github.event_name == 'pull_request' }}
        steps:
            - uses: 'actions/checkout@v4'
              name: 'Checkout'

            - name: 'Evaluation'
              env:
                  REPOSITORY: ${{ github.repository }}
                  RUN_ID: ${{ github.run_id }}
                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
              run: |
                  # Check if project-jobs was successful. Fail for any other status, including skipped. 
                  result="${{ needs.project-jobs.result }}"
                  if [[ $result != "success" ]]; then
                    echo "Generating CI jobs was not successful."
                    exit 1
                  fi

                  node .github/workflows/scripts/evaluate-jobs-conclusions.js

    alert-on-failure:
        name: 'Report results on Slack'
        runs-on: 'ubuntu-20.04'
        needs:
            [
                'project-jobs',
                'project-lint-jobs',
                'project-default-test-jobs',
                'project-e2e-test-jobs',
                'project-api-test-jobs',
                'project-performance-test-jobs',
            ]
        if: ${{ always() && github.event_name != 'pull_request'  }}
        steps:
            - uses: 'actions/checkout@v4'
              name: 'Checkout'

            - uses: './.github/actions/setup-woocommerce-monorepo'
              name: 'Setup Monorepo'
              with:
                  php-version: false

            - name: 'Send messages for failed jobs'
              env:
                  SLACK_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }}
                  SLACK_CHANNEL: ${{ secrets.TEST_REPORTS_SLACK_CHANNEL }}
                  HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
                  INPUT_TRIGGER: ${{ inputs.trigger }}
              run: |
                  COMMIT_MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`

                  pnpm utils slack-test-report -c "${{ needs.project-jobs.result }}" -r "$INPUT_TRIGGER Build jobs matrix" -m "$COMMIT_MESSAGE"
                  pnpm utils slack-test-report -c "${{ needs.project-lint-jobs.result }}" -r "$INPUT_TRIGGER Linting" -m "$COMMIT_MESSAGE"
                  pnpm utils slack-test-report -c "${{ needs.project-default-test-jobs.result }}" -r "$INPUT_TRIGGER Tests" -m "$COMMIT_MESSAGE"
                  pnpm utils slack-test-report -c "${{ needs.project-e2e-test-jobs.result }}" -r "$INPUT_TRIGGER e2e tests" -m "$COMMIT_MESSAGE"
                  pnpm utils slack-test-report -c "${{ needs.project-api-test-jobs.result }}" -r "$INPUT_TRIGGER Api tests" -m "$COMMIT_MESSAGE"
                  pnpm utils slack-test-report -c "${{ needs.project-performance-test-jobs.result }}" -r "$INPUT_TRIGGER Performance tests" -m "$COMMIT_MESSAGE"

    e2e-test-reports:
        name: 'Report e2e tests results'
        needs: [project-e2e-test-jobs]
        if: ${{ ! cancelled() && needs.project-e2e-test-jobs.result != 'skipped' }}
        runs-on: ubuntu-latest

        steps:
            - uses: actions/checkout@v4

            - name: 'Install Allure CLI'
              env:
                  DESTINATION_PATH: ../
              run: ./.github/workflows/scripts/install-allure.sh

            - name: 'Download blob reports from artifacts'
              uses: actions/download-artifact@v4
              with:
                  path: ./out
                  pattern: all-blob-e2e-reports-*
                  run-id: project-e2e-test-jobs
                  merge-multiple: true

            - name: 'Generate Allure report'
              id: generate_allure_report
              run: allure generate --clean ./out/allure-results --output ./out/allure-report

            - name: 'Archive reports'
              uses: actions/upload-artifact@v4
              with:
                  name: e2e-test-report
                  path: ./out
                  if-no-files-found: ignore
                  retention-days: 5

            - name: 'Send workflow dispatch'
              env:
                  GH_TOKEN: ${{ secrets.REPORTS_TOKEN }}
                  PR_NUMBER: ${{ github.event.pull_request.number }}
                  RUN_ID: ${{ github.run_id }}
              run: |
                  if [ "$GITHUB_EVENT_NAME" == pull_request ]; then
                    gh workflow run publish-test-reports-pr.yml \
                      -f run_id=$RUN_ID \
                      -f e2e_artifact=e2e-test-report \
                      -f pr_number=$PR_NUMBER \
                      -f commit_sha=$GITHUB_SHA \
                      -f s3_root=public \
                      --repo woocommerce/woocommerce-test-reports
                  elif [ "$GITHUB_EVENT_NAME" == push ]; then
                    gh workflow run publish-test-reports-trunk-merge.yml \
                      -f run_id=$RUN_ID \
                      -f artifact=e2e-test-report \
                      -f pr_number=$PR_NUMBER \
                      -f commit_sha=$GITHUB_SHA \
                      -f test_type="e2e" \
                      --repo woocommerce/woocommerce-test-reports
                  else
                    echo "No report will be created for '$GITHUB_EVENT_NAME' event"
                  fi

            - name: 'Add annotation to workflow run linking to PR'
              if: ${{ github.event_name == 'pull_request' }}
              run: |
                  echo "::notice::🔗🔗 The e2e report for this run is available at https://woocommerce.github.io/woocommerce-test-reports/pr/${{ github.event.pull_request.number }}/e2e"

    api-test-reports:
        name: 'Report API tests results'
        needs: [project-api-test-jobs]
        if: ${{ ! cancelled() && needs.project-api-test-jobs.result != 'skipped'}}
        runs-on: ubuntu-latest

        steps:
            - uses: actions/checkout@v4

            - name: 'Install Allure CLI'
              env:
                  DESTINATION_PATH: ../
              run: ./.github/workflows/scripts/install-allure.sh

            - name: 'Download blob reports from artifacts'
              uses: actions/download-artifact@v4
              with:
                  path: ./out/allure-results
                  pattern: all-blob-api-reports-*
                  run-id: project-api-test-jobs
                  merge-multiple: true

            - name: 'Generate Allure report'
              id: generate_allure_report
              run: allure generate --clean ./out/allure-results --output ./out/allure-report

            - name: 'Archive reports'
              uses: actions/upload-artifact@v4
              with:
                  name: api-test-report
                  path: ./out
                  if-no-files-found: ignore
                  retention-days: 5

            - name: 'Publish reports'
              env:
                  GH_TOKEN: ${{ secrets.REPORTS_TOKEN }}
                  PR_NUMBER: ${{ github.event.pull_request.number }}
                  RUN_ID: ${{ github.run_id }}
              run: |
                  if [ "$GITHUB_EVENT_NAME" == pull_request ]; then
                    gh workflow run publish-test-reports-pr.yml \
                      -f run_id=$RUN_ID \
                      -f api_artifact=api-test-report \
                      -f pr_number=$PR_NUMBER \
                      -f commit_sha=$GITHUB_SHA \
                      -f s3_root=public \
                      --repo woocommerce/woocommerce-test-reports
                  elif [ "$GITHUB_EVENT_NAME" == push ]; then
                    gh workflow run publish-test-reports-trunk-merge.yml \
                      -f run_id=$RUN_ID \
                      -f artifact=api-test-report \
                      -f pr_number=$PR_NUMBER \
                      -f commit_sha=$GITHUB_SHA \
                      -f test_type="api" \
                      --repo woocommerce/woocommerce-test-reports
                  else
                    echo "No report will be created for '$GITHUB_EVENT_NAME' event"
                  fi