name: Run tests against PR on: pull_request: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: E2E_PLAYWRIGHT: true jobs: e2e-tests-run: name: Runs E2E tests. runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - uses: ./.github/actions/cache-deps with: workflow_name: pr-build-and-e2e-tests workflow_cache: ${{ secrets.WORKFLOW_CACHE }} - name: Setup PHP uses: shivammathur/setup-php@v2 with: php-version: '7.4' - name: Install and Build uses: ./.github/actions/install-build - name: Load docker images and start containers. working-directory: plugins/woocommerce run: pnpm exec wc-e2e docker:up - name: Run Puppeteer E2E tests. if: fromJSON(env.E2E_PLAYWRIGHT) != true id: run-puppeteer-e2e-tests working-directory: plugins/woocommerce env: WC_E2E_SCREENSHOTS: 1 E2E_SLACK_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }} E2E_SLACK_CHANNEL: ${{ secrets.E2E_SLACK_CHANNEL }} run: pnpm exec wc-e2e test:e2e - name: Archive Puppeteer E2E test results if: | always() && fromJSON(env.E2E_PLAYWRIGHT) != true && ( steps.run-puppeteer-e2e-tests.conclusion != 'cancelled' || steps.run-puppeteer-e2e-tests.conclusion != 'skipped' ) uses: actions/upload-artifact@v3 with: name: pptr-e2e-test-results path: plugins/woocommerce/tests/e2e/test-results.json if-no-files-found: ignore retention-days: 5 - name: Download and install Chromium browser. if: fromJSON(env.E2E_PLAYWRIGHT) working-directory: plugins/woocommerce run: pnpx playwright install chromium - name: Run Playwright E2E tests. timeout-minutes: 60 id: run_playwright_e2e_tests if: fromJSON(env.E2E_PLAYWRIGHT) working-directory: plugins/woocommerce run: pnpx playwright test --config=e2e/playwright.config.js - name: Generate Playwright E2E Test report. id: generate_e2e_report if: | always() && fromJSON(env.E2E_PLAYWRIGHT) && ( steps.run_playwright_e2e_tests.conclusion != 'cancelled' || steps.run_playwright_e2e_tests.conclusion != 'skipped' ) working-directory: plugins/woocommerce run: pnpx allure generate --clean e2e/allure-results --output e2e/allure-report - name: Archive Playwright E2E test report if: | always() && fromJSON(env.E2E_PLAYWRIGHT) && steps.generate_e2e_report.conclusion == 'success' uses: actions/upload-artifact@v3 with: name: e2e-test-report---pr-${{ github.event.number }} path: | plugins/woocommerce/e2e/allure-results plugins/woocommerce/e2e/allure-report if-no-files-found: ignore retention-days: 5 - name: Archive Puppeteer E2E test screenshots uses: actions/upload-artifact@v3 if: | always() && ( fromJSON(env.E2E_PLAYWRIGHT) != true ) with: name: E2E Screenshots path: plugins/woocommerce/tests/e2e/screenshots if-no-files-found: ignore retention-days: 5 api-tests-run: name: Runs API tests. runs-on: ubuntu-20.04 env: API_TEST_REPORT_DIR: ${{ github.workspace }}/api-test-report steps: - uses: actions/checkout@v3 - uses: ./.github/actions/cache-deps with: workflow_name: pr-build-and-e2e-tests workflow_cache: ${{ secrets.WORKFLOW_CACHE }} - name: Setup PHP uses: shivammathur/setup-php@v2 with: php-version: '7.4' - name: Install and Build uses: ./.github/actions/install-build - name: Load docker images and start containers. working-directory: plugins/woocommerce run: pnpm exec wc-e2e docker:up - name: Run tests command. working-directory: plugins/woocommerce env: BASE_URL: http://localhost:8086 USER_KEY: admin USER_SECRET: password run: pnpm exec wc-api-tests test api - name: Archive API test report if: always() uses: actions/upload-artifact@v3 with: name: api-test-report---pr-${{ github.event.number }} path: | ${{ env.API_TEST_REPORT_DIR }}/allure-results ${{ env.API_TEST_REPORT_DIR }}/allure-report retention-days: 5 k6-tests-run: name: Runs k6 Performance tests runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - uses: ./.github/actions/cache-deps with: workflow_name: pr-build-and-e2e-tests workflow_cache: ${{ secrets.WORKFLOW_CACHE }} - name: Setup PHP uses: shivammathur/setup-php@v2 with: php-version: '7.4' - name: Install and Build uses: ./.github/actions/install-build - name: Workaround to use initialization file with prepopulated data. working-directory: plugins/woocommerce/tests/e2e/docker run: | cp init-sample-products.sh initialize.sh - name: Load docker images and start containers. working-directory: plugins/woocommerce run: pnpm exec wc-e2e docker:up - name: Install k6 run: | curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1 - name: Run k6 tests run: | ./k6 run plugins/woocommerce/tests/performance/tests/gh-action-pr-requests.js test-summary: name: Post test results if: | always() && ! github.event.pull_request.head.repo.fork && ( contains( needs.*.result, 'success' ) || contains( needs.*.result, 'failure' ) ) runs-on: ubuntu-20.04 needs: [api-tests-run, e2e-tests-run] steps: - name: Create dirs run: | mkdir -p repo mkdir -p artifacts/api mkdir -p artifacts/e2e mkdir -p output - name: Checkout code uses: actions/checkout@v3 with: path: repo - name: Download API test report artifact uses: actions/download-artifact@v3 with: name: api-test-report---pr-${{ github.event.number }} path: artifacts/api - name: Download Playwright E2E test report artifact if: fromJSON(env.E2E_PLAYWRIGHT) uses: actions/download-artifact@v3 with: name: e2e-test-report---pr-${{ github.event.number }} path: artifacts/e2e - name: Download Puppeteer E2E test report artifact if: fromJSON(env.E2E_PLAYWRIGHT) != true uses: actions/download-artifact@v3 with: name: pptr-e2e-test-results path: artifacts/e2e - name: Prepare test summary id: prepare-test-summary uses: actions/github-script@v6 env: API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json E2E_PPTR_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/test-results.json PR_NUMBER: ${{ github.event.number }} SHA: ${{ github.event.pull_request.head.sha }} with: result-encoding: string script: | const script = require( './repo/.github/workflows/scripts/prepare-test-summary.js' ) return await script( { core } ) - name: Find PR comment by github-actions[bot] uses: peter-evans/find-comment@v2 id: find-comment with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' body-includes: Test Results Summary - name: Create or update PR comment uses: peter-evans/create-or-update-comment@v2 with: comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body: ${{ steps.prepare-test-summary.outputs.result }} edit-mode: replace publish-test-reports: name: Publish test reports if: | always() && ! github.event.pull_request.head.repo.fork && ( contains( needs.*.result, 'success' ) || contains( needs.*.result, 'failure' ) ) runs-on: ubuntu-20.04 needs: [api-tests-run, e2e-tests-run, k6-tests-run] env: GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} PR_NUMBER: ${{ github.event.number }} RUN_ID: ${{ github.run_id }} COMMIT_SHA: ${{ github.event.pull_request.head.sha }} steps: - name: Publish API test report if: fromJSON(env.E2E_PLAYWRIGHT) != true env: ARTIFACT_NAME: api-test-report---pr-${{ github.event.number }} run: | gh workflow run publish-report.yml \ -f test_workflow=pr \ -f test_type=api \ -f run_id=$RUN_ID \ -f artifact_name=$ARTIFACT_NAME \ -f pr_number=$PR_NUMBER \ -f commit_sha=$COMMIT_SHA \ --repo woocommerce/woocommerce-test-reports - name: Publish test reports if: fromJSON(env.E2E_PLAYWRIGHT) env: API_ARTIFACT: api-test-report---pr-${{ github.event.number }} E2E_ARTIFACT: e2e-test-report---pr-${{ github.event.number }} run: | gh workflow run publish-test-reports-pr.yml \ -f run_id=$RUN_ID \ -f api_artifact=$API_ARTIFACT \ -f e2e_artifact=$E2E_ARTIFACT \ -f pr_number=$PR_NUMBER \ -f commit_sha=$COMMIT_SHA \ -f s3_root=public \ --repo woocommerce/woocommerce-test-reports