woocommerce/.github/workflows/smoke-test-daily.yml

name: Smoke test daily
on:
    schedule:
        - cron: '25 3 * * *'
    workflow_dispatch:

env:
    API_ARTIFACT: api-daily--run-${{ github.run_number }}
    E2E_ARTIFACT: e2e-daily--run-${{ github.run_number }}
    FORCE_COLOR: 1

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

permissions: {}

jobs:
    # api-tests:
    #     name: API tests on nightly build
    #     runs-on: ubuntu-20.04
    #     permissions:
    #         contents: read
    #     env:
    #         ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results
    #         ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report
    #         BASE_URL: ${{ secrets.SMOKE_TEST_URL }}
    #         ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }}
    #         ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }}
    #         ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }}
    #         CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }}
    #         CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }}
    #         DEFAULT_TIMEOUT_OVERRIDE: 120000
    #     steps:
    #         - uses: actions/checkout@v3

    #         - name: Setup WooCommerce Monorepo
    #           uses: ./.github/actions/setup-woocommerce-monorepo
    #           with:
    #               install-filters: woocommerce
    #               build: false

    #         - name: Download and install Chromium browser.
    #           working-directory: plugins/woocommerce
    #           run: pnpm exec playwright install chromium

    #         - name: Run 'Update WooCommerce' test.
    #           working-directory: plugins/woocommerce
    #           env:
    #               UPDATE_WC: nightly
    #           run: pnpm exec playwright test --config=tests/e2e-pw/daily.playwright.config.js update-woocommerce.spec.js

    #         - name: Run API tests.
    #           working-directory: plugins/woocommerce
    #           env:
    #               USER_KEY: ${{ secrets.SMOKE_TEST_ADMIN_USER }}
    #               USER_SECRET: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }}
    #           run: pnpm exec playwright test --config=tests/api-core-tests/playwright.config.js hello.test.js

    #         - name: Generate API Test report.
    #           if: success() || failure()
    #           working-directory: plugins/woocommerce
    #           run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}

    #         - name: Archive API test report
    #           if: success() || failure()
    #           uses: actions/upload-artifact@v3
    #           with:
    #               name: ${{ env.API_ARTIFACT }}
    #               path: |
    #                   ${{ env.ALLURE_RESULTS_DIR }}
    #                   ${{ env.ALLURE_REPORT_DIR }}
    #               if-no-files-found: ignore
    #               retention-days: 5

    # e2e-tests:
    #     name: E2E tests on nightly build
    #     runs-on: ubuntu-20.04
    #     permissions:
    #         contents: read
    #     needs: [api-tests]
    #     env:
    #         ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }}
    #         ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }}
    #         ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }}
    #         ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
    #         ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
    #         BASE_URL: ${{ secrets.SMOKE_TEST_URL }}
    #         CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }}
    #         CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }}
    #         DEFAULT_TIMEOUT_OVERRIDE: 120000

    #     steps:
    #         - uses: actions/checkout@v3

    #         - name: Setup WooCommerce Monorepo
    #           uses: ./.github/actions/setup-woocommerce-monorepo
    #           with:
    #               install-filters: woocommerce
    #               build: false

    #         - name: Download and install Chromium browser.
    #           working-directory: plugins/woocommerce
    #           run: pnpm exec playwright install chromium

    #         - name: Run E2E tests.
    #           timeout-minutes: 60
    #           working-directory: plugins/woocommerce
    #           env:
    #               E2E_MAX_FAILURES: 25
    #               RESET_SITE: true
    #           run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js

    #         - name: Generate Playwright E2E Test report.
    #           if: success() || failure()
    #           working-directory: plugins/woocommerce
    #           run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}

    #         - name: Archive E2E test report
    #           if: success() || failure()
    #           uses: actions/upload-artifact@v3
    #           with:
    #               name: ${{ env.E2E_ARTIFACT }}
    #               path: |
    #                   ${{ env.ALLURE_RESULTS_DIR }}
    #                   ${{ env.ALLURE_REPORT_DIR }}
    #               if-no-files-found: ignore
    #               retention-days: 5

    # k6-tests:
    #     name: k6 tests on nightly build
    #     runs-on: ubuntu-20.04
    #     permissions:
    #         contents: read
    #     needs: [api-tests]
    #     if: success() || failure()
    #     steps:
    #         - uses: actions/checkout@v3

    #         - name: Setup WooCommerce Monorepo
    #           uses: ./.github/actions/setup-woocommerce-monorepo
    #           with:
    #               install-filters: woocommerce
    #               build: false

    #         - name: Download and install Chromium browser.
    #           working-directory: plugins/woocommerce
    #           run: pnpm exec playwright install chromium

    #         - name: Update performance test site with E2E test
    #           working-directory: plugins/woocommerce
    #           env:
    #               BASE_URL: ${{ secrets.SMOKE_TEST_PERF_URL }}/
    #               ADMIN_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
    #               ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
    #               CUSTOMER_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
    #               CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
    #               UPDATE_WC: nightly
    #               DEFAULT_TIMEOUT_OVERRIDE: 120000
    #           run: |
    #               pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js update-woocommerce.spec.js
    #           continue-on-error: true

    #         - name: Install k6
    #           run: |
    #               curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1

    #         - name: Run k6 smoke tests
    #           env:
    #               URL: ${{ secrets.SMOKE_TEST_PERF_URL }}
    #               HOST: ${{ secrets.SMOKE_TEST_PERF_HOST }}
    #               A_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
    #               A_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
    #               C_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }}
    #               C_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }}
    #               P_ID: 22733
    #           run: |
    #               ./k6 run plugins/woocommerce/tests/performance/tests/gh-action-daily-ext-requests.js

    test-plugins:
        name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed
        runs-on: ubuntu-20.04
        permissions:
            contents: read
        # mytodo uncomment
        # needs: [api-tests]
        env:
            USE_WP_ENV: 1
            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results
            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report
        strategy:
            fail-fast: false
            matrix:
                include:
                    # mytodo uncomment
                    # - plugin: 'WooCommerce Payments'
                    #   repo: 'automattic/woocommerce-payments'
                    # - plugin: 'WooCommerce PayPal Payments'
                    #   repo: 'woocommerce/woocommerce-paypal-payments'
                    # - plugin: 'WooCommerce Shipping & Tax'
                    #   repo: 'automattic/woocommerce-services'
                    # - plugin: 'WooCommerce Subscriptions'
                    #   repo: WC_SUBSCRIPTIONS_REPO
                    #   private: true
                    - plugin: 'Gutenberg'
                      repo: 'WordPress/gutenberg'
                    - plugin: 'Gutenberg - Nightly'
                      repo: 'bph/gutenberg'
        steps:
            - uses: actions/checkout@v3

            - name: Setup WooCommerce Monorepo
              uses: ./.github/actions/setup-woocommerce-monorepo
              with:
                  build-filters: woocommerce

            - name: Launch wp-env e2e environment
              working-directory: plugins/woocommerce
              run: pnpm env:test --filter=woocommerce

            - name: Download and install Chromium browser.
              working-directory: plugins/woocommerce
              run: pnpm exec playwright install chromium

            - name: Run 'Upload plugin' test
              working-directory: plugins/woocommerce
              env:
                  PLUGIN_REPOSITORY: ${{ matrix.private && secrets[matrix.repo] || matrix.repo }}
                  PLUGIN_NAME: ${{ matrix.plugin }}
                  GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }}
              run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js upload-plugin.spec.js

            - name: Run the rest of E2E tests
              working-directory: plugins/woocommerce
              env:
                  E2E_MAX_FAILURES: 15
              # mytodo remove basic.spec
              run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js basic.spec.js

            - name: Generate E2E Test report.
              if: success() || failure()
              working-directory: plugins/woocommerce
              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}

            - name: Archive E2E test report
              if: success() || failure()
              uses: actions/upload-artifact@v3
              with:
                  name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
                  path: |
                      ${{ env.ALLURE_RESULTS_DIR }}
                      ${{ env.ALLURE_REPORT_DIR }}
                  if-no-files-found: ignore
                  retention-days: 5

    # trunk-results:
    #     name: Publish report on smoke tests on nightly build
    #     if: |
    #         ( success() || failure() ) &&
    #         ! github.event.pull_request.head.repo.fork
    #     runs-on: ubuntu-20.04
    #     permissions:
    #         contents: read
    #     needs: [e2e-tests, test-plugins, k6-tests]
    #     steps:
    #         - name: Create dirs
    #           run: |
    #               mkdir -p repo
    #               mkdir -p artifacts/api
    #               mkdir -p artifacts/e2e
    #               mkdir -p output

    #         - name: Checkout code
    #           uses: actions/checkout@v3
    #           with:
    #               path: repo

    #         - name: Download API test report artifact
    #           uses: actions/download-artifact@v3
    #           with:
    #               name: ${{ env.API_ARTIFACT }}
    #               path: artifacts/api

    #         - name: Download E2E test report artifact
    #           uses: actions/download-artifact@v3
    #           with:
    #               name: ${{ env.E2E_ARTIFACT }}
    #               path: artifacts/e2e

    #         - name: Post test summary
    #           uses: actions/github-script@v6
    #           env:
    #               API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json
    #               E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json
    #           with:
    #               result-encoding: string
    #               script: |
    #                   const script = require( './repo/.github/workflows/scripts/prepare-test-summary-daily.js' )
    #                   return await script( { core } )

    #         - name: Publish report
    #           env:
    #               GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
    #               RUN_ID: ${{ github.run_id }}
    #           run: |
    #               gh workflow run publish-test-reports-daily.yml \
    #                 -f run_id=$RUN_ID \
    #                 -f api_artifact="$API_ARTIFACT" \
    #                 -f e2e_artifact="$E2E_ARTIFACT" \
    #                 -f s3_root=public \
    #                 --repo woocommerce/woocommerce-test-reports

    # plugins-results:
    #     name: Publish report on Smoke tests on trunk with plugins
    #     if: |
    #         ( success() || failure() ) &&
    #         ! github.event.pull_request.head.repo.fork
    #     runs-on: ubuntu-20.04
    #     # needs: [e2e-tests, test-plugins, k6-tests]
    #     needs: [test-plugins] # mytodo revert to the above
    #     env:
    #         GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }}
    #         RUN_ID: ${{ github.run_id }}
    #         ARTIFACT: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
    #     strategy:
    #         fail-fast: false
    #         matrix:
    #             include:
    #                 # mytodo uncomment
    #                 # - plugin: 'WooCommerce Payments'
    #                 #   repo: 'automattic/woocommerce-payments'
    #                 # - plugin: 'WooCommerce PayPal Payments'
    #                 #   repo: 'woocommerce/woocommerce-paypal-payments'
    #                 # - plugin: 'WooCommerce Shipping & Tax'
    #                 #   repo: 'automattic/woocommerce-services'
    #                 # - plugin: 'WooCommerce Subscriptions'
    #                 #   repo: WC_SUBSCRIPTIONS_REPO
    #                 #   private: true
    #                 - plugin: 'Gutenberg'
    #                   repo: 'WordPress/gutenberg'
    #                 - plugin: 'Gutenberg - Nightly'
    #                   repo: 'bph/gutenberg'
    #     steps:
    #         - name: Download test report artifact
    #           uses: actions/download-artifact@v3
    #           with:
    #               name: ${{ env.ARTIFACT }}

    #         # mytodo fix this, same slug between Gutenberg and its nightly
    #         - name: Get slug
    #           id: get-slug
    #           uses: actions/github-script@v6
    #           with:
    #               result-encoding: string
    #               script: return "${{ matrix.repo }}".split( '/' ).pop()

    #         - name: Publish reports
    #           run: |
    #               gh workflow run publish-test-reports-daily-plugins.yml \
    #                 -f run_id=$RUN_ID \
    #                 -f artifact="${{ env.ARTIFACT }}" \
    #                 -f plugin="${{ matrix.plugin }}" \
    #                 -f slug="${{ steps.get-slug.outputs.result }}" \
    #                 -f s3_root=public \
    #                 --repo woocommerce/woocommerce-test-reports