420 lines
17 KiB
YAML
420 lines
17 KiB
YAML
name: Smoke test daily
|
|
on:
|
|
schedule:
|
|
- cron: '25 3 * * *'
|
|
workflow_dispatch:
|
|
|
|
env:
|
|
API_ARTIFACT: api-daily--run-${{ github.run_number }}
|
|
E2E_ARTIFACT: e2e-daily--run-${{ github.run_number }}
|
|
FORCE_COLOR: 1
|
|
PLUGIN_SLACK_BLOCKS_ARTIFACT: plugin-blocks
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
permissions: {}
|
|
|
|
jobs:
|
|
api-tests:
|
|
name: API tests on nightly build
|
|
runs-on: ubuntu-20.04
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
test-result: ${{ steps.run-api-composite-action.outputs.result }}
|
|
env:
|
|
ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results
|
|
ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Setup WooCommerce Monorepo
|
|
uses: ./.github/actions/setup-woocommerce-monorepo
|
|
|
|
- name: Setup local test environment
|
|
uses: ./.github/actions/tests/setup-local-test-environment
|
|
with:
|
|
test-type: api
|
|
|
|
- name: Run API tests
|
|
id: run-api-composite-action
|
|
uses: ./.github/actions/tests/run-api-tests
|
|
with:
|
|
report-name: ${{ env.API_ARTIFACT }}
|
|
env:
|
|
BASE_URL: http://localhost:8086
|
|
USER_KEY: admin
|
|
USER_SECRET: password
|
|
GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }}
|
|
|
|
- name: Generate Playwright API Test report.
|
|
id: generate_api_report
|
|
if: |
|
|
always() &&
|
|
(
|
|
steps.run-api-composite-action.conclusion != 'cancelled' ||
|
|
steps.run-api-composite-action.conclusion != 'skipped'
|
|
)
|
|
working-directory: plugins/woocommerce
|
|
run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
|
|
|
|
- name: Archive Playwright API test report
|
|
if: |
|
|
always() &&
|
|
steps.generate_api_report.conclusion == 'success'
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: ${{ env.API_ARTIFACT }}
|
|
path: |
|
|
${{ env.ALLURE_RESULTS_DIR }}
|
|
${{ env.ALLURE_REPORT_DIR }}
|
|
if-no-files-found: ignore
|
|
retention-days: 20
|
|
|
|
e2e-tests:
|
|
name: E2E tests on nightly build
|
|
runs-on: ubuntu-20.04
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
test-result: ${{ steps.run_playwright_e2e_tests.outputs.result }}
|
|
# needs: [api-tests]
|
|
env:
|
|
ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
|
|
ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Setup WooCommerce Monorepo
|
|
uses: ./.github/actions/setup-woocommerce-monorepo
|
|
|
|
- name: Setup local test environment
|
|
uses: ./.github/actions/tests/setup-local-test-environment
|
|
with:
|
|
test-type: e2e
|
|
|
|
- name: Download and install Chromium browser.
|
|
working-directory: plugins/woocommerce
|
|
run: pnpm exec playwright install chromium
|
|
|
|
- name: Run E2E tests
|
|
timeout-minutes: 60
|
|
id: run_playwright_e2e_tests
|
|
env:
|
|
USE_WP_ENV: 1
|
|
E2E_MAX_FAILURES: 15
|
|
FORCE_COLOR: 1
|
|
ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
|
|
ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
|
|
working-directory: plugins/woocommerce
|
|
run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js
|
|
|
|
- name: Generate Test report.
|
|
if: success() || ( failure() && steps.run_playwright_e2e_tests.conclusion == 'failure' )
|
|
working-directory: plugins/woocommerce
|
|
run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
|
|
|
|
- name: Archive test report
|
|
if: success() || ( failure() && steps.run_playwright_e2e_tests.conclusion == 'failure' )
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: ${{ env.E2E_ARTIFACT }}
|
|
path: |
|
|
${{ env.ALLURE_RESULTS_DIR }}
|
|
${{ env.ALLURE_REPORT_DIR }}
|
|
retention-days: 20
|
|
|
|
k6-tests:
|
|
name: k6 tests on nightly build
|
|
runs-on: ubuntu-20.04
|
|
permissions:
|
|
contents: read
|
|
needs: [api-tests]
|
|
if: success() || failure()
|
|
outputs:
|
|
test-result: ${{ steps.run-k6-tests.conclusion }}
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Setup WooCommerce Monorepo
|
|
uses: ./.github/actions/setup-woocommerce-monorepo
|
|
with:
|
|
install-filters: woocommerce
|
|
build: false
|
|
|
|
- name: Update performance test site with E2E test
|
|
id: update-perf-site
|
|
continue-on-error: true
|
|
uses: ./.github/actions/tests/run-e2e-tests
|
|
with:
|
|
report-name: k6-daily-update-site--run-${{ github.run_number }}
|
|
tests: update-woocommerce.spec.js
|
|
env:
|
|
ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results
|
|
ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report
|
|
BASE_URL: ${{ secrets.SMOKE_TEST_PERF_URL }}/
|
|
ADMIN_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
|
|
ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
|
|
CUSTOMER_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
|
|
CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
|
|
DEFAULT_TIMEOUT_OVERRIDE: 120000
|
|
GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }}
|
|
UPDATE_WC: nightly
|
|
|
|
- name: Install k6
|
|
run: |
|
|
curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1
|
|
|
|
- name: Run k6 smoke tests
|
|
id: run-k6-tests
|
|
env:
|
|
URL: ${{ secrets.SMOKE_TEST_PERF_URL }}
|
|
HOST: ${{ secrets.SMOKE_TEST_PERF_HOST }}
|
|
A_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
|
|
A_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
|
|
C_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
|
|
C_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
|
|
P_ID: 22733
|
|
run: |
|
|
./k6 run plugins/woocommerce/tests/performance/tests/gh-action-daily-ext-requests.js
|
|
|
|
test-plugins:
|
|
name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed
|
|
runs-on: ubuntu-20.04
|
|
permissions:
|
|
contents: read
|
|
needs: [api-tests]
|
|
env:
|
|
ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results
|
|
ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- plugin: 'WooCommerce Payments'
|
|
repo: 'automattic/woocommerce-payments'
|
|
slug: woocommerce-payments
|
|
- plugin: 'WooCommerce PayPal Payments'
|
|
repo: 'woocommerce/woocommerce-paypal-payments'
|
|
slug: woocommerce-paypal-payments
|
|
- plugin: 'WooCommerce Shipping & Tax'
|
|
repo: 'automattic/woocommerce-services'
|
|
slug: woocommerce-services
|
|
- plugin: 'Woo Subscriptions'
|
|
repo: WC_SUBSCRIPTIONS_REPO
|
|
private: true
|
|
slug: woocommerce-subscriptions
|
|
- plugin: 'Gutenberg'
|
|
repo: 'WordPress/gutenberg'
|
|
slug: gutenberg
|
|
- plugin: 'Gutenberg - Nightly'
|
|
repo: 'bph/gutenberg'
|
|
slug: gutenberg-nightly
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Setup WooCommerce Monorepo
|
|
uses: ./.github/actions/setup-woocommerce-monorepo
|
|
with:
|
|
build-filters: woocommerce
|
|
|
|
- name: Setup local test environment
|
|
uses: ./.github/actions/tests/setup-local-test-environment
|
|
with:
|
|
test-type: e2e
|
|
|
|
- name: Run 'Upload plugin' test
|
|
id: run-upload-plugin-test
|
|
uses: ./.github/actions/tests/run-e2e-tests
|
|
with:
|
|
report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
|
|
tests: upload-plugin.spec.js
|
|
env:
|
|
PLUGIN_REPOSITORY: ${{ matrix.private && secrets[matrix.repo] || matrix.repo }}
|
|
PLUGIN_NAME: ${{ matrix.plugin }}
|
|
PLUGIN_SLUG: ${{ matrix.slug }}
|
|
GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }}
|
|
|
|
- name: Run the rest of E2E tests
|
|
id: run-e2e-composite-action
|
|
timeout-minutes: 60
|
|
uses: ./.github/actions/tests/run-e2e-tests
|
|
with:
|
|
playwright-config: ignore-plugin-tests.playwright.config.js
|
|
report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
|
|
env:
|
|
E2E_MAX_FAILURES: 15
|
|
|
|
- name: Create context block and save as JSON file
|
|
if: success() || failure()
|
|
id: create-block-json
|
|
uses: actions/github-script@v6
|
|
with:
|
|
script: |
|
|
const script = require( './.github/actions/tests/slack-summary-daily/scripts/create-blocks-plugin-tests.js' )
|
|
script( { core } );
|
|
env:
|
|
UPLOAD_RESULT: ${{ steps.run-upload-plugin-test.outputs.result }}
|
|
E2E_RESULT: ${{ steps.run-e2e-composite-action.outputs.result }}
|
|
PLUGIN_NAME: ${{ matrix.plugin }}
|
|
PLUGIN_SLUG: ${{ matrix.slug }}
|
|
|
|
- name: Upload JSON file as artifact
|
|
if: success() || failure()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }}
|
|
path: ${{ steps.create-block-json.outputs.path }}
|
|
|
|
trunk-results:
|
|
name: Publish report on smoke tests on nightly build
|
|
if: |
|
|
( success() || failure() ) &&
|
|
! github.event.pull_request.head.repo.fork
|
|
runs-on: ubuntu-20.04
|
|
permissions:
|
|
contents: read
|
|
needs: [e2e-tests, test-plugins, k6-tests]
|
|
steps:
|
|
- name: Create dirs
|
|
run: |
|
|
mkdir -p repo
|
|
mkdir -p artifacts/api
|
|
mkdir -p artifacts/e2e
|
|
mkdir -p output
|
|
|
|
- name: Checkout code
|
|
uses: actions/checkout@v3
|
|
with:
|
|
path: repo
|
|
|
|
- name: Download API test report artifact
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: ${{ env.API_ARTIFACT }}
|
|
path: artifacts/api
|
|
|
|
- name: Download E2E test report artifact
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: ${{ env.E2E_ARTIFACT }}
|
|
path: artifacts/e2e
|
|
|
|
- name: Post test summary
|
|
uses: actions/github-script@v6
|
|
env:
|
|
API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json
|
|
E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json
|
|
with:
|
|
result-encoding: string
|
|
script: |
|
|
const script = require( './repo/.github/workflows/scripts/prepare-test-summary-daily.js' )
|
|
return await script( { core } )
|
|
|
|
- name: Publish report
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
|
|
RUN_ID: ${{ github.run_id }}
|
|
run: |
|
|
gh workflow run publish-test-reports-daily.yml \
|
|
-f run_id=$RUN_ID \
|
|
-f api_artifact="$API_ARTIFACT" \
|
|
-f e2e_artifact="$E2E_ARTIFACT" \
|
|
-f s3_root=public \
|
|
--repo woocommerce/woocommerce-test-reports
|
|
|
|
plugins-results:
|
|
name: Publish report on Smoke tests on trunk with plugins
|
|
if: |
|
|
( success() || failure() ) &&
|
|
( needs.test-plugins.result != 'skipped' ) &&
|
|
! github.event.pull_request.head.repo.fork
|
|
runs-on: ubuntu-20.04
|
|
needs: [e2e-tests, test-plugins, k6-tests]
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
|
|
RUN_ID: ${{ github.run_id }}
|
|
ARTIFACT: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- plugin: 'WooCommerce Payments'
|
|
slug: woocommerce-payments
|
|
- plugin: 'WooCommerce PayPal Payments'
|
|
slug: woocommerce-paypal-payments
|
|
- plugin: 'WooCommerce Shipping & Tax'
|
|
slug: woocommerce-services
|
|
- plugin: 'Woo Subscriptions'
|
|
slug: woocommerce-subscriptions
|
|
- plugin: 'Gutenberg'
|
|
slug: gutenberg
|
|
- plugin: 'Gutenberg - Nightly'
|
|
slug: gutenberg-nightly
|
|
steps:
|
|
- name: Download test report artifact
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: ${{ env.ARTIFACT }}
|
|
|
|
- name: Publish reports
|
|
run: |
|
|
gh workflow run publish-test-reports-daily-plugins.yml \
|
|
-f run_id=$RUN_ID \
|
|
-f artifact="${{ env.ARTIFACT }}" \
|
|
-f plugin="${{ matrix.plugin }}" \
|
|
-f slug="${{ matrix.slug }}" \
|
|
-f s3_root=public \
|
|
--repo woocommerce/woocommerce-test-reports
|
|
|
|
post-slack-summary:
|
|
name: Post Slack summary
|
|
runs-on: ubuntu-20.04
|
|
permissions:
|
|
contents: read
|
|
if: |
|
|
github.ref_name == 'trunk' &&
|
|
success() || (
|
|
failure() && contains( needs.*.result, 'failure' )
|
|
)
|
|
needs:
|
|
- api-tests
|
|
- e2e-tests
|
|
- k6-tests
|
|
- test-plugins
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Download Slack blocks from plugin tests
|
|
if: needs.test-plugins.result != 'skipped'
|
|
id: download-plugin-blocks
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }}
|
|
path: /tmp/plugin-blocks
|
|
|
|
- name: Construct Slack payload
|
|
id: construct-slack-payload
|
|
uses: actions/github-script@v6
|
|
with:
|
|
script: |
|
|
const script = require('./.github/actions/tests/slack-summary-daily/scripts/construct-slack-payload.js');
|
|
await script( { context, core, github } );
|
|
env:
|
|
API_RESULT: ${{ needs.api-tests.outputs.test-result }}
|
|
E2E_RESULT: ${{ needs.e2e-tests.outputs.test-result || needs.e2e-tests.result }}
|
|
k6_RESULT: ${{ needs.k6-tests.outputs.test-result || needs.k6-tests.result }}
|
|
PLUGINS_BLOCKS_PATH: ${{ steps.download-plugin-blocks.outputs.download-path }}
|
|
PLUGIN_TESTS_RESULT: ${{ needs.test-plugins.result }}
|
|
|
|
- name: Send Slack message
|
|
id: send-slack-message
|
|
uses: slackapi/slack-github-action@v1.23.0
|
|
with:
|
|
channel-id: ${{ secrets.DAILY_TEST_SLACK_CHANNEL }}
|
|
payload: ${{ steps.construct-slack-payload.outputs.payload }}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }}
|