woocommerce/.github/workflows/smoke-test-daily.yml

name: Smoke test daily
on:
    schedule:
        - cron: '25 3 * * *'
    workflow_dispatch:

env:
    API_ARTIFACT: api-daily--run-${{ github.run_number }}
    E2E_ARTIFACT: e2e-daily--run-${{ github.run_number }}
    FORCE_COLOR: 1
    PLUGIN_SLACK_BLOCKS_ARTIFACT: plugin-blocks

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

permissions: {}

jobs:
    api-tests:
        name: API tests on nightly build
        runs-on: ubuntu-20.04
        permissions:
            contents: read
        outputs:
            test-result: ${{ steps.run-api-composite-action.outputs.result }}
        env:
            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results
            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report
            BASE_URL: ${{ secrets.SMOKE_TEST_URL }}
            API_BASE_URL: ${{ secrets.SMOKE_TEST_API_URL }}
            ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }}
            ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }}
            ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }}
            CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }}
            CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }}
            DEFAULT_TIMEOUT_OVERRIDE: 120000
            UPDATE_WC: 'nightly'
        steps:
            - uses: actions/checkout@v3

            - name: Setup WooCommerce Monorepo
              uses: ./.github/actions/setup-woocommerce-monorepo
              with:
                  install-filters: woocommerce
                  build: false

            - name: Run API tests
              id: run-api-composite-action
              uses: ./.github/actions/tests/run-api-tests
              with:
                  report-name: ${{ env.API_ARTIFACT }}
              env:
                  USER_KEY: ${{ secrets.SMOKE_TEST_ADMIN_USER }}
                  USER_SECRET: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }}
                  GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }}
                  UPDATE_WC: nightly

    e2e-tests:
        name: E2E tests on nightly build
        runs-on: ubuntu-20.04
        permissions:
            contents: read
        outputs:
            test-result: ${{ steps.run-e2e-composite-action.outputs.result }}
        #  needs: [api-tests]
        env:
            ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }}
            ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }}
            ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }}
            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
            BASE_URL: ${{ secrets.SMOKE_TEST_URL }}
            CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }}
            CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }}
            DEFAULT_TIMEOUT_OVERRIDE: 120000
        steps:
            - uses: actions/checkout@v3

            - name: Setup WooCommerce Monorepo
              uses: ./.github/actions/setup-woocommerce-monorepo
              with:
                  install-filters: woocommerce
                  build: false

            - name: Run E2E tests
              id: run-e2e-composite-action
              timeout-minutes: 60
              uses: ./.github/actions/tests/run-e2e-tests
              with:
                  report-name: ${{ env.E2E_ARTIFACT }}
              env:
                  ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }}
                  ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }}
                  ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }}
                  ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
                  ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
                  BASE_URL: ${{ secrets.SMOKE_TEST_URL }}
                  CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }}
                  CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }}
                  DEFAULT_TIMEOUT_OVERRIDE: 120000
                  E2E_MAX_FAILURES: 25
                  RESET_SITE: true

    k6-tests:
        name: k6 tests on nightly build
        runs-on: ubuntu-20.04
        permissions:
            contents: read
        needs: [api-tests]
        if: success() || failure()
        outputs:
            test-result: ${{ steps.run-k6-tests.conclusion }}
        steps:
            - uses: actions/checkout@v3

            - name: Setup WooCommerce Monorepo
              uses: ./.github/actions/setup-woocommerce-monorepo
              with:
                  install-filters: woocommerce
                  build: false

            - name: Update performance test site with E2E test
              id: update-perf-site
              continue-on-error: true
              uses: ./.github/actions/tests/run-e2e-tests
              with:
                  report-name: k6-daily-update-site--run-${{ github.run_number }}
                  tests: update-woocommerce.spec.js
              env:
                  ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results
                  ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report
                  BASE_URL: ${{ secrets.SMOKE_TEST_PERF_URL }}/
                  ADMIN_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
                  ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
                  CUSTOMER_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
                  CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
                  DEFAULT_TIMEOUT_OVERRIDE: 120000
                  GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }}
                  UPDATE_WC: nightly

            - name: Install k6
              run: |
                  curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1

            - name: Run k6 smoke tests
              id: run-k6-tests
              env:
                  URL: ${{ secrets.SMOKE_TEST_PERF_URL }}
                  HOST: ${{ secrets.SMOKE_TEST_PERF_HOST }}
                  A_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
                  A_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
                  C_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
                  C_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
                  P_ID: 22733
              run: |
                  ./k6 run plugins/woocommerce/tests/performance/tests/gh-action-daily-ext-requests.js

    test-plugins:
        name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed
        runs-on: ubuntu-20.04
        permissions:
            contents: read
        needs: [api-tests]
        env:
            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results
            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report
        strategy:
            fail-fast: false
            matrix:
                include:
                    - plugin: 'WooCommerce Payments'
                      repo: 'automattic/woocommerce-payments'
                      slug: woocommerce-payments
                    - plugin: 'WooCommerce PayPal Payments'
                      repo: 'woocommerce/woocommerce-paypal-payments'
                      slug: woocommerce-paypal-payments
                    - plugin: 'WooCommerce Shipping & Tax'
                      repo: 'automattic/woocommerce-services'
                      slug: woocommerce-services
                    - plugin: 'WooCommerce Subscriptions'
                      repo: WC_SUBSCRIPTIONS_REPO
                      private: true
                      slug: woocommerce-subscriptions
                    - plugin: 'Gutenberg'
                      repo: 'WordPress/gutenberg'
                      slug: gutenberg
                    - plugin: 'Gutenberg - Nightly'
                      repo: 'bph/gutenberg'
                      slug: gutenberg-nightly
        steps:
            - uses: actions/checkout@v3

            - name: Setup WooCommerce Monorepo
              uses: ./.github/actions/setup-woocommerce-monorepo
              with:
                  build-filters: woocommerce

            - name: Setup local test environment
              uses: ./.github/actions/tests/setup-local-test-environment
              with:
                  test-type: e2e

            - name: Run 'Upload plugin' test
              id: run-upload-plugin-test
              uses: ./.github/actions/tests/run-e2e-tests
              with:
                  report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
                  tests: upload-plugin.spec.js
              env:
                  PLUGIN_REPOSITORY: ${{ matrix.private && secrets[matrix.repo] || matrix.repo }}
                  PLUGIN_NAME: ${{ matrix.plugin }}
                  GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }}

            - name: Run the rest of E2E tests
              id: run-e2e-composite-action
              timeout-minutes: 60
              uses: ./.github/actions/tests/run-e2e-tests
              with:
                  playwright-config: ignore-plugin-tests.playwright.config.js
                  report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
              env:
                  E2E_MAX_FAILURES: 15

            - name: Create context block and save as JSON file
              if: success() || failure()
              id: create-block-json
              uses: actions/github-script@v6
              with:
                  script: |
                      const script = require( './.github/actions/tests/slack-summary-daily/scripts/create-blocks-plugin-tests.js' )
                      script( { core } );
              env:
                  UPLOAD_RESULT: ${{ steps.run-upload-plugin-test.outputs.result }}
                  E2E_RESULT: ${{ steps.run-e2e-composite-action.outputs.result }}
                  PLUGIN_NAME: ${{ matrix.plugin }}
                  PLUGIN_SLUG: ${{ matrix.slug }}

            - name: Upload JSON file as artifact
              if: success() || failure()
              uses: actions/upload-artifact@v3
              with:
                  name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }}
                  path: ${{ steps.create-block-json.outputs.path }}

    trunk-results:
        name: Publish report on smoke tests on nightly build
        if: |
            ( success() || failure() ) &&
            ! github.event.pull_request.head.repo.fork
        runs-on: ubuntu-20.04
        permissions:
            contents: read
        needs: [e2e-tests, test-plugins, k6-tests]
        steps:
            - name: Create dirs
              run: |
                  mkdir -p repo
                  mkdir -p artifacts/api
                  mkdir -p artifacts/e2e
                  mkdir -p output

            - name: Checkout code
              uses: actions/checkout@v3
              with:
                  path: repo

            - name: Download API test report artifact
              uses: actions/download-artifact@v3
              with:
                  name: ${{ env.API_ARTIFACT }}
                  path: artifacts/api

            - name: Download E2E test report artifact
              uses: actions/download-artifact@v3
              with:
                  name: ${{ env.E2E_ARTIFACT }}
                  path: artifacts/e2e

            - name: Post test summary
              uses: actions/github-script@v6
              env:
                  API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json
                  E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json
              with:
                  result-encoding: string
                  script: |
                      const script = require( './repo/.github/workflows/scripts/prepare-test-summary-daily.js' )
                      return await script( { core } )

            - name: Publish report
              env:
                  GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
                  RUN_ID: ${{ github.run_id }}
              run: |
                  gh workflow run publish-test-reports-daily.yml \
                    -f run_id=$RUN_ID \
                    -f api_artifact="$API_ARTIFACT" \
                    -f e2e_artifact="$E2E_ARTIFACT" \
                    -f s3_root=public \
                    --repo woocommerce/woocommerce-test-reports

    plugins-results:
        name: Publish report on Smoke tests on trunk with plugins
        if: |
            ( success() || failure() ) && 
            ( needs.test-plugins.result != 'skipped' ) &&
            ! github.event.pull_request.head.repo.fork
        runs-on: ubuntu-20.04
        needs: [e2e-tests, test-plugins, k6-tests]
        env:
            GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
            RUN_ID: ${{ github.run_id }}
            ARTIFACT: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
        strategy:
            fail-fast: false
            matrix:
                include:
                    - plugin: 'WooCommerce Payments'
                      slug: woocommerce-payments
                    - plugin: 'WooCommerce PayPal Payments'
                      slug: woocommerce-paypal-payments
                    - plugin: 'WooCommerce Shipping & Tax'
                      slug: woocommerce-services
                    - plugin: 'WooCommerce Subscriptions'
                      slug: woocommerce-subscriptions
                    - plugin: 'Gutenberg'
                      slug: gutenberg
                    - plugin: 'Gutenberg - Nightly'
                      slug: gutenberg-nightly
        steps:
            - name: Download test report artifact
              uses: actions/download-artifact@v3
              with:
                  name: ${{ env.ARTIFACT }}

            - name: Publish reports
              run: |
                  gh workflow run publish-test-reports-daily-plugins.yml \
                    -f run_id=$RUN_ID \
                    -f artifact="${{ env.ARTIFACT }}" \
                    -f plugin="${{ matrix.plugin }}" \
                    -f slug="${{ matrix.slug }}" \
                    -f s3_root=public \
                    --repo woocommerce/woocommerce-test-reports

    post-slack-summary:
        name: Post Slack summary
        runs-on: ubuntu-20.04
        permissions:
            contents: read
        if: |
            success() || (
              failure() && contains( needs.*.result, 'failure' )
            )
        needs:
            - api-tests
            - e2e-tests
            - k6-tests
            - test-plugins
        steps:
            - uses: actions/checkout@v3

            - name: Download Slack blocks from plugin tests
              if: needs.test-plugins.result != 'skipped'
              id: download-plugin-blocks
              uses: actions/download-artifact@v3
              with:
                  name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }}
                  path: /tmp/plugin-blocks

            - name: Construct Slack payload
              id: construct-slack-payload
              uses: actions/github-script@v6
              with:
                  script: |
                      const script = require('./.github/actions/tests/slack-summary-daily/scripts/construct-slack-payload.js');
                      await script( { context, core, github } );
              env:
                  API_RESULT: ${{ needs.api-tests.outputs.test-result }}
                  E2E_RESULT: ${{ needs.e2e-tests.outputs.test-result || needs.e2e-tests.result }}
                  k6_RESULT: ${{ needs.k6-tests.outputs.test-result || needs.k6-tests.result }}
                  PLUGINS_BLOCKS_PATH: ${{ steps.download-plugin-blocks.outputs.download-path }}
                  PLUGIN_TESTS_RESULT: ${{ needs.test-plugins.result }}

            - name: Send Slack message
              id: send-slack-message
              uses: slackapi/slack-github-action@v1.23.0
              with:
                  channel-id: ${{ secrets.DAILY_TEST_SLACK_CHANNEL }}
                  payload: ${{ steps.construct-slack-payload.outputs.payload }}
              env:
                  SLACK_BOT_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }}