name: Smoke test daily on: schedule: - cron: '25 3 * * *' workflow_dispatch: env: API_ARTIFACT: api-daily--run-${{ github.run_number }} E2E_ARTIFACT: e2e-daily--run-${{ github.run_number }} FORCE_COLOR: 1 PLUGIN_SLACK_BLOCKS_ARTIFACT: plugin-blocks concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: {} jobs: api-tests: name: API tests on nightly build runs-on: ubuntu-20.04 permissions: contents: read outputs: test-result: ${{ steps.run-api-composite-action.outputs.result }} env: ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report BASE_URL: ${{ secrets.SMOKE_TEST_URL }} API_BASE_URL: ${{ secrets.SMOKE_TEST_API_URL }} ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} DEFAULT_TIMEOUT_OVERRIDE: 120000 UPDATE_WC: 'nightly' steps: - uses: actions/checkout@v3 - name: Setup WooCommerce Monorepo uses: ./.github/actions/setup-woocommerce-monorepo with: install-filters: woocommerce build: false - name: Update site to nightly version uses: ./.github/actions/tests/run-e2e-tests with: report-name: ${{ env.API_ARTIFACT }} tests: update-woocommerce.spec.js env: GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} UPDATE_WC: nightly - name: Run API tests id: run-api-composite-action uses: ./.github/actions/tests/run-api-tests with: report-name: ${{ env.API_ARTIFACT }} env: USER_KEY: ${{ secrets.SMOKE_TEST_ADMIN_USER }} USER_SECRET: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} e2e-tests: name: E2E tests on nightly build runs-on: ubuntu-20.04 permissions: contents: read outputs: test-result: ${{ steps.run-e2e-composite-action.outputs.result }} # needs: [api-tests] env: ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results BASE_URL: ${{ secrets.SMOKE_TEST_URL }} CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} DEFAULT_TIMEOUT_OVERRIDE: 120000 steps: - uses: actions/checkout@v3 - name: Setup WooCommerce Monorepo uses: ./.github/actions/setup-woocommerce-monorepo with: install-filters: woocommerce build: false - name: Run E2E tests id: run-e2e-composite-action timeout-minutes: 60 uses: ./.github/actions/tests/run-e2e-tests with: report-name: ${{ env.E2E_ARTIFACT }} env: ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results BASE_URL: ${{ secrets.SMOKE_TEST_URL }} CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} DEFAULT_TIMEOUT_OVERRIDE: 120000 E2E_MAX_FAILURES: 25 RESET_SITE: true k6-tests: name: k6 tests on nightly build runs-on: ubuntu-20.04 permissions: contents: read needs: [api-tests] if: success() || failure() outputs: test-result: ${{ steps.run-k6-tests.conclusion }} steps: - uses: actions/checkout@v3 - name: Setup WooCommerce Monorepo uses: ./.github/actions/setup-woocommerce-monorepo with: install-filters: woocommerce build: false - name: Update performance test site with E2E test id: update-perf-site continue-on-error: true uses: ./.github/actions/tests/run-e2e-tests with: report-name: k6-daily-update-site--run-${{ github.run_number }} tests: update-woocommerce.spec.js env: ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report BASE_URL: ${{ secrets.SMOKE_TEST_PERF_URL }}/ ADMIN_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} CUSTOMER_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} DEFAULT_TIMEOUT_OVERRIDE: 120000 GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} UPDATE_WC: nightly - name: Install k6 run: | curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1 - name: Run k6 smoke tests id: run-k6-tests env: URL: ${{ secrets.SMOKE_TEST_PERF_URL }} HOST: ${{ secrets.SMOKE_TEST_PERF_HOST }} A_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} A_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} C_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} C_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} P_ID: 22733 run: | ./k6 run plugins/woocommerce/tests/performance/tests/gh-action-daily-ext-requests.js test-plugins: name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed runs-on: ubuntu-20.04 permissions: contents: read needs: [api-tests] env: ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report strategy: fail-fast: false matrix: include: - plugin: 'WooCommerce Payments' repo: 'automattic/woocommerce-payments' slug: woocommerce-payments - plugin: 'WooCommerce PayPal Payments' repo: 'woocommerce/woocommerce-paypal-payments' slug: woocommerce-paypal-payments - plugin: 'WooCommerce Shipping & Tax' repo: 'automattic/woocommerce-services' slug: woocommerce-services - plugin: 'WooCommerce Subscriptions' repo: WC_SUBSCRIPTIONS_REPO private: true slug: woocommerce-subscriptions - plugin: 'Gutenberg' repo: 'WordPress/gutenberg' slug: gutenberg - plugin: 'Gutenberg - Nightly' repo: 'bph/gutenberg' slug: gutenberg-nightly steps: - uses: actions/checkout@v3 - name: Setup WooCommerce Monorepo uses: ./.github/actions/setup-woocommerce-monorepo with: build-filters: woocommerce - name: Setup local test environment uses: ./.github/actions/tests/setup-local-test-environment with: test-type: e2e - name: Run 'Upload plugin' test id: run-upload-plugin-test uses: ./.github/actions/tests/run-e2e-tests with: report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) tests: upload-plugin.spec.js env: PLUGIN_REPOSITORY: ${{ matrix.private && secrets[matrix.repo] || matrix.repo }} PLUGIN_NAME: ${{ matrix.plugin }} GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} - name: Run the rest of E2E tests id: run-e2e-composite-action timeout-minutes: 60 uses: ./.github/actions/tests/run-e2e-tests with: playwright-config: ignore-plugin-tests.playwright.config.js report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) env: E2E_MAX_FAILURES: 15 - name: Create context block and save as JSON file if: success() || failure() id: create-block-json uses: actions/github-script@v6 with: script: | const script = require( './.github/actions/tests/slack-summary-daily/scripts/create-blocks-plugin-tests.js' ) script( { core } ); env: UPLOAD_RESULT: ${{ steps.run-upload-plugin-test.outputs.result }} E2E_RESULT: ${{ steps.run-e2e-composite-action.outputs.result }} PLUGIN_NAME: ${{ matrix.plugin }} PLUGIN_SLUG: ${{ matrix.slug }} - name: Upload JSON file as artifact if: success() || failure() uses: actions/upload-artifact@v3 with: name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }} path: ${{ steps.create-block-json.outputs.path }} trunk-results: name: Publish report on smoke tests on nightly build if: | ( success() || failure() ) && ! github.event.pull_request.head.repo.fork runs-on: ubuntu-20.04 permissions: contents: read needs: [e2e-tests, test-plugins, k6-tests] steps: - name: Create dirs run: | mkdir -p repo mkdir -p artifacts/api mkdir -p artifacts/e2e mkdir -p output - name: Checkout code uses: actions/checkout@v3 with: path: repo - name: Download API test report artifact uses: actions/download-artifact@v3 with: name: ${{ env.API_ARTIFACT }} path: artifacts/api - name: Download E2E test report artifact uses: actions/download-artifact@v3 with: name: ${{ env.E2E_ARTIFACT }} path: artifacts/e2e - name: Post test summary uses: actions/github-script@v6 env: API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json with: result-encoding: string script: | const script = require( './repo/.github/workflows/scripts/prepare-test-summary-daily.js' ) return await script( { core } ) - name: Publish report env: GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} RUN_ID: ${{ github.run_id }} run: | gh workflow run publish-test-reports-daily.yml \ -f run_id=$RUN_ID \ -f api_artifact="$API_ARTIFACT" \ -f e2e_artifact="$E2E_ARTIFACT" \ -f s3_root=public \ --repo woocommerce/woocommerce-test-reports plugins-results: name: Publish report on Smoke tests on trunk with plugins if: | ( success() || failure() ) && ( needs.test-plugins.result != 'skipped' ) && ! github.event.pull_request.head.repo.fork runs-on: ubuntu-20.04 needs: [e2e-tests, test-plugins, k6-tests] env: GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} RUN_ID: ${{ github.run_id }} ARTIFACT: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) strategy: fail-fast: false matrix: include: - plugin: 'WooCommerce Payments' slug: woocommerce-payments - plugin: 'WooCommerce PayPal Payments' slug: woocommerce-paypal-payments - plugin: 'WooCommerce Shipping & Tax' slug: woocommerce-services - plugin: 'WooCommerce Subscriptions' slug: woocommerce-subscriptions - plugin: 'Gutenberg' slug: gutenberg - plugin: 'Gutenberg - Nightly' slug: gutenberg-nightly steps: - name: Download test report artifact uses: actions/download-artifact@v3 with: name: ${{ env.ARTIFACT }} - name: Publish reports run: | gh workflow run publish-test-reports-daily-plugins.yml \ -f run_id=$RUN_ID \ -f artifact="${{ env.ARTIFACT }}" \ -f plugin="${{ matrix.plugin }}" \ -f slug="${{ matrix.slug }}" \ -f s3_root=public \ --repo woocommerce/woocommerce-test-reports post-slack-summary: name: Post Slack summary runs-on: ubuntu-20.04 permissions: contents: read if: | success() || ( failure() && contains( needs.*.result, 'failure' ) ) needs: - api-tests - e2e-tests - k6-tests - test-plugins steps: - uses: actions/checkout@v3 - name: Download Slack blocks from plugin tests if: needs.test-plugins.result != 'skipped' id: download-plugin-blocks uses: actions/download-artifact@v3 with: name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }} path: /tmp/plugin-blocks - name: Construct Slack payload id: construct-slack-payload uses: actions/github-script@v6 with: script: | const script = require('./.github/actions/tests/slack-summary-daily/scripts/construct-slack-payload.js'); await script( { context, core, github } ); env: API_RESULT: ${{ needs.api-tests.outputs.test-result }} E2E_RESULT: ${{ needs.e2e-tests.outputs.test-result || needs.e2e-tests.result }} k6_RESULT: ${{ needs.k6-tests.outputs.test-result || needs.k6-tests.result }} PLUGINS_BLOCKS_PATH: ${{ steps.download-plugin-blocks.outputs.download-path }} PLUGIN_TESTS_RESULT: ${{ needs.test-plugins.result }} - name: Send Slack message id: send-slack-message uses: slackapi/slack-github-action@v1.23.0 with: channel-id: ${{ secrets.DAILY_TEST_SLACK_CHANNEL }} payload: ${{ steps.construct-slack-payload.outputs.payload }} env: SLACK_BOT_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }}