woocommerce/.github/workflows/pr-build-and-e2e-tests.yml

name: Run tests against PR
on:
    workflow_dispatch:
    pull_request:
        paths-ignore:
            - '**.md'
            - '**/changelog/**'
            - '**/.wireit/**'
            - 'docs/**'
            - 'plugins/woo-ai/**'
            - 'plugins/woocommerce/.wordpress-org/**'
            - 'plugins/woocommerce/i18n/**'
            - 'plugins/woocommerce-beta-tester/**'
            - 'tools/**'

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

permissions: {}

jobs:
    e2e-tests-run:
        name: Runs E2E tests in matrix.
        runs-on: ubuntu-latest
        timeout-minutes: 60
        strategy:
            fail-fast: false
            matrix:
                shard:
                    [
                        { number: 1, name: 1/5 },
                        { number: 2, name: 2/5 },
                        { number: 3, name: 3/5 },
                        { number: 4, name: 4/5 },
                        { number: 5, name: 5/5 },
                    ]
        permissions:
            contents: read
        env:
            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
        outputs:
            E2E_GRAND_TOTAL: ${{ steps.count_e2e_total.outputs.E2E_GRAND_TOTAL }}
        steps:
            - uses: actions/checkout@v3

            - name: Setup WooCommerce Monorepo
              uses: ./.github/actions/setup-woocommerce-monorepo
              with:
                  install: '@woocommerce/plugin-woocommerce...'
                  build: '@woocommerce/plugin-woocommerce'

            - name: Load docker images and start containers.
              working-directory: plugins/woocommerce
              env:
                  WP_ENV_PHP_VERSION: 7.4
              run: pnpm env:test

            - name: Download and install Chromium browser.
              working-directory: plugins/woocommerce
              run: pnpm exec playwright install chromium

            - name: Get total number of Playwright E2E tests to be run.
              id: count_e2e_total
              working-directory: plugins/woocommerce
              run: |
                  TOTAL_STR=$(pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js --list | grep "Total:")
                  NO_PREFIX=${TOTAL_STR#*"Total: "}
                  COUNT=${NO_PREFIX%" tests in"*}
                  echo "E2E_GRAND_TOTAL=$COUNT" >> $GITHUB_OUTPUT

            - name: Run Playwright tests
              working-directory: plugins/woocommerce
              env:
                  USE_WP_ENV: 1
                  E2E_MAX_FAILURES: 15
                  FORCE_COLOR: 1
              id: run_playwright_e2e_tests
              run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js --shard ${{ matrix.shard.name }}

            - name: Upload reports to GitHub Actions Artifacts
              if: always()
              uses: actions/upload-artifact@v4
              with:
                  name: all-blob-reports-${{ matrix.shard.number }}
                  path: ${{ env.ALLURE_RESULTS_DIR }}
                  retention-days: 1
                  compression-level: 9

    e2e-tests-success:
        name: Evaluate e2e tests results
        runs-on: ubuntu-latest
        needs: e2e-tests-run
        if: ${{ always() }}
        steps:
            - run: |
                result="${{ needs.e2e-tests-run.result }}"
                if [[ $result != "success" && $result != "skipped" ]]; then
                  echo "One or more e2e tests have failed!"
                exit 1
                fi
                echo "e2e tests have completed successfully."

    merge-reports:
        name: Merge e2e test reports
        # Merge reports after playwright-tests, even if some shards have failed
        if: always()
        needs: [e2e-tests-success]

        runs-on: ubuntu-latest
        env:
            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
        steps:
            - uses: actions/checkout@v3
            - name: Setup WooCommerce Monorepo
              uses: ./.github/actions/setup-woocommerce-monorepo
              with:
                  install: '@woocommerce/plugin-woocommerce...'
                  build: '@woocommerce/plugin-woocommerce'

            - name: Download blob reports from GitHub Actions Artifacts
              uses: actions/download-artifact@v4
              with:
                  path: ${{ env.ALLURE_RESULTS_DIR }}
                  pattern: all-blob-reports-*
                  run-id: e2e-tests-run
                  merge-multiple: true

            - name: Generate Playwright E2E Test report.
              id: generate_e2e_report
              working-directory: plugins/woocommerce
              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}

            - name: Archive Playwright E2E test report
              if: |
                  always() &&
                  steps.generate_e2e_report.conclusion == 'success'
              uses: actions/upload-artifact@v4
              with:
                  name: e2e-test-report---pr-${{ github.event.number }}
                  path: |
                      ${{ env.ALLURE_RESULTS_DIR }}
                      ${{ env.ALLURE_REPORT_DIR }}
                  if-no-files-found: ignore
                  retention-days: 5

    api-tests-run:
        name: Runs API tests.
        runs-on: ubuntu-20.04
        permissions:
            contents: read
        env:
            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results
            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report
        steps:
            - uses: actions/checkout@v3

            - name: Setup WooCommerce Monorepo
              uses: ./.github/actions/setup-woocommerce-monorepo
              with:
                  install: '@woocommerce/plugin-woocommerce...'
                  build: '@woocommerce/plugin-woocommerce'

            - name: Load docker images and start containers.
              working-directory: plugins/woocommerce
              env:
                  ENABLE_HPOS: 0
              run: pnpm --filter=@woocommerce/plugin-woocommerce env:test

            - name: Run Playwright API tests.
              id: run_playwright_api_tests
              working-directory: plugins/woocommerce
              env:
                  BASE_URL: http://localhost:8086
                  USER_KEY: admin
                  USER_SECRET: password
              run: pnpm exec playwright test --config=tests/api-core-tests/playwright.config.js

            - name: Generate Playwright API Test report.
              id: generate_api_report
              if: |
                  always() &&
                  (
                    steps.run_playwright_api_tests.conclusion != 'cancelled' ||
                    steps.run_playwright_api_tests.conclusion != 'skipped' 
                  )
              working-directory: plugins/woocommerce
              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
            - name: Archive Playwright API test report
              if: |
                  always() &&
                  steps.generate_api_report.conclusion == 'success'
              uses: actions/upload-artifact@v4
              with:
                  name: api-test-report---pr-${{ github.event.number }}
                  path: |
                      ${{ env.ALLURE_RESULTS_DIR }}
                      ${{ env.ALLURE_REPORT_DIR }}
                  if-no-files-found: ignore
                  retention-days: 5

    k6-tests-run:
        name: Runs k6 Performance tests
        if: github.event.pull_request.user.login != 'github-actions[bot]'
        runs-on: ubuntu-20.04
        permissions:
            contents: read
        steps:
            - uses: actions/checkout@v3

            - name: Setup WooCommerce Monorepo
              uses: ./.github/actions/setup-woocommerce-monorepo
              with:
                  install: '@woocommerce/plugin-woocommerce...'
                  build: '@woocommerce/plugin-woocommerce'

            - name: Load docker images and start containers.
              working-directory: plugins/woocommerce
              env:
                  ENABLE_HPOS: 0
              run: |
                  pnpm --filter=@woocommerce/plugin-woocommerce env:dev
                  pnpm --filter=@woocommerce/plugin-woocommerce env:performance-init

            - name: Install k6
              run: |
                  curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1

            - name: Run k6 tests
              run: |
                  ./k6 run plugins/woocommerce/tests/performance/tests/gh-action-pr-requests.js

    test-summary:
        name: Post test results
        if: |
            always() && 
            ! github.event.pull_request.head.repo.fork &&
            github.event.pull_request.user.login != 'github-actions[bot]' &&
            (
              contains( needs.*.result, 'success' ) ||
              contains( needs.*.result, 'failure' )
            )
        runs-on: ubuntu-20.04
        needs: [api-tests-run, merge-reports]
        permissions:
            contents: read
            issues: write
            pull-requests: write
        env:
            E2E_GRAND_TOTAL: ${{needs.e2e-tests-run.outputs.E2E_GRAND_TOTAL}}
        steps:
            - name: Create dirs
              run: |
                  mkdir -p repo
                  mkdir -p artifacts/api 
                  mkdir -p artifacts/e2e
                  mkdir -p output

            - name: Checkout code
              uses: actions/checkout@v3
              with:
                  path: repo

            - name: Download API test report artifact
              uses: actions/download-artifact@v4
              with:
                  name: api-test-report---pr-${{ github.event.number }}
                  path: artifacts/api

            - name: Download Playwright E2E test report artifact
              uses: actions/download-artifact@v4
              with:
                  name: e2e-test-report---pr-${{ github.event.number }}
                  path: artifacts/e2e

            - name: Prepare test summary
              id: prepare-test-summary
              uses: actions/github-script@v6
              env:
                  API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json
                  E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json
                  PR_NUMBER: ${{ github.event.number }}
                  SHA: ${{ github.event.pull_request.head.sha }}
              with:
                  result-encoding: string
                  script: |
                      const script = require( './repo/.github/workflows/scripts/prepare-test-summary.js' )
                      return await script( { core } )

            - name: Find PR comment by github-actions[bot]
              uses: peter-evans/find-comment@034abe94d3191f9c89d870519735beae326f2bdb
              id: find-comment
              with:
                  issue-number: ${{ github.event.pull_request.number }}
                  comment-author: 'github-actions[bot]'
                  body-includes: Test Results Summary

            - name: Create or update PR comment
              uses: peter-evans/create-or-update-comment@67dcc547d311b736a8e6c5c236542148a47adc3d
              with:
                  comment-id: ${{ steps.find-comment.outputs.comment-id }}
                  issue-number: ${{ github.event.pull_request.number }}
                  body: ${{ steps.prepare-test-summary.outputs.result }}
                  edit-mode: replace

    publish-test-reports:
        name: Publish test reports
        if: |
            always() && 
            ! github.event.pull_request.head.repo.fork &&
            github.event.pull_request.user.login != 'github-actions[bot]' &&
            (
              contains( needs.*.result, 'success' ) ||
              contains( needs.*.result, 'failure' )
            )
        runs-on: ubuntu-20.04
        needs: [api-tests-run, merge-reports, k6-tests-run]
        env:
            GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
            PR_NUMBER: ${{ github.event.number }}
            RUN_ID: ${{ github.run_id }}
            COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
        steps:
            - name: Publish test reports
              env:
                  API_ARTIFACT: api-test-report---pr-${{ github.event.number }}
                  E2E_ARTIFACT: e2e-test-report---pr-${{ github.event.number }}
              run: |
                  gh workflow run publish-test-reports-pr.yml \
                    -f run_id=$RUN_ID \
                    -f api_artifact=$API_ARTIFACT \
                    -f e2e_artifact=$E2E_ARTIFACT \
                    -f pr_number=$PR_NUMBER \
                    -f commit_sha=$COMMIT_SHA \
                    -f s3_root=public \
                    --repo woocommerce/woocommerce-test-reports