From 1fee0439d3d0e5ab114057e19c94d6f03301ffa6 Mon Sep 17 00:00:00 2001 From: Adrian Moldovan Date: Fri, 12 Apr 2024 17:46:01 +0300 Subject: [PATCH] [testing workflows] Move metrics job to ci.yml (#46471) * Move deprecated workflows to deprecated folder * Configure new test job for metrics * Add changelog * Update start command * Update start command * Archive metrics results * Fix script path * Fix name in archive results condition * Add the actual run command for pull_request event * Run with pnpm --filter * Test run for push event * Revert test conditions * Add WP_ARTIFACTS_PATH env variable * Update path to readme.txt * Revert github event conditions * Add CODEVITALS_PROJECT_TOKEN env * Set WP_ARTIFACTS_PATH on job level * Add blocks templates in the changes list --- .github/workflows/ci.yml | 12 +++++++ .../workflows/{ => deprecated}/metrics.yml | 0 .../pr-build-and-e2e-tests.yml | 6 ++-- .../{ => deprecated}/smoke-test-pr-merge.yml | 6 ++-- .github/workflows/scripts/run-metrics.sh | 31 +++++++++++++++++++ .../changelog/test-move-metrics-tests-to-ci | 4 +++ plugins/woocommerce/package.json | 16 ++++++++++ 7 files changed, 69 insertions(+), 6 deletions(-) rename .github/workflows/{ => deprecated}/metrics.yml (100%) rename .github/workflows/{ => deprecated}/pr-build-and-e2e-tests.yml (99%) rename .github/workflows/{ => deprecated}/smoke-test-pr-merge.yml (99%) create mode 100755 .github/workflows/scripts/run-metrics.sh create mode 100644 plugins/woocommerce/changelog/test-move-metrics-tests-to-ci diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d706149db0..026f2bcc96c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,9 @@ jobs: fail-fast: false matrix: include: ${{ fromJSON( needs.project-jobs.outputs.performance-test-jobs ) }} + env: + WP_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts + steps: - uses: 'actions/checkout@v4' name: 'Checkout' @@ -193,7 +196,16 @@ jobs: run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}' - name: 'Run tests' + env: + CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}' + + - name: 'Archive metrics results' + if: ${{ success() && matrix.name == 'Metrics' }} + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: metrics-results + path: ${{ env.WP_ARTIFACTS_PATH }}/*.performance-results*.json evaluate-project-jobs: # In order to add a required status check we need a consistent job that we can grab onto. diff --git a/.github/workflows/metrics.yml b/.github/workflows/deprecated/metrics.yml similarity index 100% rename from .github/workflows/metrics.yml rename to .github/workflows/deprecated/metrics.yml diff --git a/.github/workflows/pr-build-and-e2e-tests.yml b/.github/workflows/deprecated/pr-build-and-e2e-tests.yml similarity index 99% rename from .github/workflows/pr-build-and-e2e-tests.yml rename to .github/workflows/deprecated/pr-build-and-e2e-tests.yml index 4edd57092fc..4f5c9462695 100644 --- a/.github/workflows/pr-build-and-e2e-tests.yml +++ b/.github/workflows/deprecated/pr-build-and-e2e-tests.yml @@ -1,9 +1,9 @@ name: Run tests against PR on: workflow_dispatch: - #pull_request: - #paths-ignore: - #- '**/changelog/**' + pull_request: + paths-ignore: + - '**/changelog/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/smoke-test-pr-merge.yml b/.github/workflows/deprecated/smoke-test-pr-merge.yml similarity index 99% rename from .github/workflows/smoke-test-pr-merge.yml rename to .github/workflows/deprecated/smoke-test-pr-merge.yml index 991cbef0aa6..39b85581c43 100644 --- a/.github/workflows/smoke-test-pr-merge.yml +++ b/.github/workflows/deprecated/smoke-test-pr-merge.yml @@ -1,9 +1,9 @@ name: Run tests against trunk after PR merge on: workflow_dispatch: - #pull_request: - #types: - #- closed + pull_request: + types: + - closed concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true diff --git a/.github/workflows/scripts/run-metrics.sh b/.github/workflows/scripts/run-metrics.sh new file mode 100755 index 00000000000..4bbe85ee470 --- /dev/null +++ b/.github/workflows/scripts/run-metrics.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -eo pipefail + +if [[ -z "$GITHUB_EVENT_NAME" ]]; then + echo "::error::GITHUB_EVENT_NAME must be set" + exit 1 +fi + +if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + echo "Comparing performance with trunk" + pnpm --filter="compare-perf" run compare perf $GITHUB_SHA trunk --tests-branch $GITHUB_SHA + +elif [[ "$GITHUB_EVENT_NAME" == "push" ]]; then + echo "Comparing performance with base branch" + # The base hash used here need to be a commit that is compatible with the current WP version + # The current one is 19f3d0884617d7ecdcf37664f648a51e2987cada + # it needs to be updated every time it becomes unsupported by the current wp-env (WP version). + # It is used as a base comparison point to avoid fluctuation in the performance metrics. + WP_VERSION=$(awk -F ': ' '/^Tested up to/{print $2}' readme.txt) + echo "WP_VERSION: $WP_VERSION" + IFS=. read -ra WP_VERSION_ARRAY <<< "$WP_VERSION" + WP_MAJOR="${WP_VERSION_ARRAY[0]}.${WP_VERSION_ARRAY[1]}" + pnpm --filter="compare-perf" run compare perf $GITHUB_SHA 19f3d0884617d7ecdcf37664f648a51e2987cada --tests-branch $GITHUB_SHA --wp-version "$WP_MAJOR" + + echo "Publish results to CodeVitals" + COMMITTED_AT=$(git show -s $GITHUB_SHA --format="%cI") + pnpm --filter="compare-perf" run log $CODEVITALS_PROJECT_TOKEN trunk $GITHUB_SHA 19f3d0884617d7ecdcf37664f648a51e2987cada $COMMITTED_AT +else + echo "Unsupported event: $GITHUB_EVENT_NAME" +fi diff --git a/plugins/woocommerce/changelog/test-move-metrics-tests-to-ci b/plugins/woocommerce/changelog/test-move-metrics-tests-to-ci new file mode 100644 index 00000000000..22b9a7c35f0 --- /dev/null +++ b/plugins/woocommerce/changelog/test-move-metrics-tests-to-ci @@ -0,0 +1,4 @@ +Significance: patch +Type: dev + +Move metrics job into ci.yml diff --git a/plugins/woocommerce/package.json b/plugins/woocommerce/package.json index 771c5d0d6a7..a611e807c70 100644 --- a/plugins/woocommerce/package.json +++ b/plugins/woocommerce/package.json @@ -59,6 +59,7 @@ "test:php": "./vendor/bin/phpunit -c ./phpunit.xml", "test:php:watch": "./vendor/bin/phpunit-watcher watch", "test:metrics": "USE_WP_ENV=1 pnpm playwright test --config=tests/metrics/playwright.config.js", + "test:metrics:ci": "../../.github/workflows/scripts/run-metrics.sh", "test:php:env": "wp-env run --env-cwd='wp-content/plugins/woocommerce' tests-cli vendor/bin/phpunit -c phpunit.xml --verbose", "test:php:env:watch": "wp-env run --env-cwd='wp-content/plugins/woocommerce' tests-cli vendor/bin/phpunit-watcher watch --verbose", "test:unit": "pnpm test:php", @@ -244,6 +245,21 @@ "testEnv": { "start": "env:perf" } + }, + { + "name": "Metrics", + "testType": "performance", + "command": "test:metrics:ci", + "changes": [ + "client/admin/config/*.json", + "composer.lock", + "includes/**/*.php", + "patterns/**/*.php", + "src/**/*.php", + "templates/**/*.php", + "templates/**/*.html", + "tests/metrics/**" + ] } ] }