From 68617e8ef1c2f4056da8d371ceebaf5818bdbc7d Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Tue, 12 Dec 2023 10:33:34 +0100 Subject: [PATCH] Performance: Add a tool and a CI workflow to compare performance between PR and trunk and track metrics on trunk --- .github/workflows/metrics.yml | 53 ++++ .gitignore | 1 + pnpm-lock.yaml | 48 ++-- pnpm-workspace.yaml | 1 + tools/compare-perf/config.js | 11 + tools/compare-perf/index.js | 41 +++ tools/compare-perf/package.json | 23 ++ tools/compare-perf/performance.js | 429 ++++++++++++++++++++++++++++++ tools/compare-perf/utils.js | 105 ++++++++ 9 files changed, 693 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/metrics.yml create mode 100644 tools/compare-perf/config.js create mode 100755 tools/compare-perf/index.js create mode 100644 tools/compare-perf/package.json create mode 100644 tools/compare-perf/performance.js create mode 100644 tools/compare-perf/utils.js diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml new file mode 100644 index 00000000000..a78e9a92a17 --- /dev/null +++ b/.github/workflows/metrics.yml @@ -0,0 +1,53 @@ +name: Metrics Tracking + +on: + pull_request: + push: + branches: [trunk] + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +permissions: {} + +jobs: + metrics: + name: Run metrics tests + runs-on: ubuntu-20.04 + permissions: + contents: read + env: + WP_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts + + steps: + - uses: actions/checkout@v3 + + - name: Setup WooCommerce Monorepo + uses: ./.github/actions/setup-woocommerce-monorepo + with: + install: '@woocommerce/plugin-woocommerce...' + build: '@woocommerce/plugin-woocommerce' + + - name: Compare performance with trunk + if: github.event_name == 'pull_request' + run: cd tools/compare-perf && pnpm run perf $GITHUB_SHA trunk --tests-branch $GITHUB_SHA + + - name: Compare performance with base branch + if: github.event_name == 'push' + # The base hash used here need to be a commit that is compatible with the current WP version + # The current one is 2f6ca66e00b3666b2567877ae67cbfb5b6ce171a + # it needs to be updated every time it becomes unsupported by the current wp-env (WP version). + # It is used as a base comparison point to avoid fluctuation in the performance metrics. + run: | + WP_VERSION=$(awk -F ': ' '/^Tested up to/{print $2}' readme.txt) + IFS=. read -ra WP_VERSION_ARRAY <<< "$WP_VERSION" + WP_MAJOR="${WP_VERSION_ARRAY[0]}.${WP_VERSION_ARRAY[1]}" + cd tools/compare-perf && pnpm run perf $GITHUB_SHA 2f6ca66e00b3666b2567877ae67cbfb5b6ce171a --tests-branch $GITHUB_SHA --wp-version "$WP_MAJOR" + + - name: Archive performance results + if: success() + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: performance-results + path: ${{ env.WP_ARTIFACTS_PATH }}/*.performance-results*.json diff --git a/.gitignore b/.gitignore index 6b73893a40f..4dedc16153c 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,4 @@ changes.json # Metrics tests /artifacts /plugins/*/artifacts +/tools/*/artifacts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b57a7029c6b..e7a909f1a25 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4610,6 +4610,24 @@ importers: specifier: 0.14.1 version: 0.14.1 + tools/compare-perf: + dependencies: + '@wordpress/env': + specifier: ^8.13.0 + version: 8.13.0 + chalk: + specifier: ^4.1.2 + version: 4.1.2 + commander: + specifier: 9.5.0 + version: 9.5.0 + inquirer: + specifier: ^7.1.0 + version: 7.3.3 + simple-git: + specifier: 3.5.0 + version: 3.5.0 + tools/create-extension: dependencies: chalk: @@ -14670,7 +14688,6 @@ packages: /@sindresorhus/is@4.6.0: resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} engines: {node: '>=10'} - dev: true /@sindresorhus/is@5.6.0: resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} @@ -18448,7 +18465,6 @@ packages: engines: {node: '>=10'} dependencies: defer-to-connect: 2.0.1 - dev: true /@szmarczak/http-timer@5.0.1: resolution: {integrity: sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==} @@ -18744,7 +18760,6 @@ packages: '@types/keyv': 3.1.4 '@types/node': 20.10.4 '@types/responselike': 1.0.3 - dev: true /@types/cheerio@0.22.35: resolution: {integrity: sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==} @@ -23250,7 +23265,6 @@ packages: yargs: 17.7.2 transitivePeerDependencies: - supports-color - dev: true /@wordpress/env@8.2.0: resolution: {integrity: sha512-MGf2TJD6MbBjDn/+feGAcXnh6ct9Y/aCjIZTtbMI+b+pXAPh0RzR7Q5D6KMgKUNcGeL6nNQmEUUSSPuzV2UWfQ==} @@ -27864,7 +27878,7 @@ packages: resolution: {integrity: sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==} dependencies: base64-js: 1.5.1 - ieee754: 1.1.13 + ieee754: 1.2.1 isarray: 1.0.0 dev: true @@ -28047,7 +28061,6 @@ packages: /cacheable-lookup@5.0.4: resolution: {integrity: sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==} engines: {node: '>=10.6.0'} - dev: true /cacheable-lookup@7.0.0: resolution: {integrity: sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==} @@ -28078,7 +28091,6 @@ packages: lowercase-keys: 2.0.0 normalize-url: 6.1.0 responselike: 2.0.1 - dev: true /call-bind@1.0.5: resolution: {integrity: sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==} @@ -28772,7 +28784,6 @@ packages: resolution: {integrity: sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==} dependencies: mimic-response: 1.0.1 - dev: true /clone-stats@1.0.0: resolution: {integrity: sha512-au6ydSpg6nsrigcZ4m8Bc9hxjeW+GJ8xh5G3BJCMt4WXe1H10UNaVOamqQTmrx1kjVuxAHIQSNU6hY4Nsn9/ag==} @@ -29269,7 +29280,6 @@ packages: /copy-dir@1.3.0: resolution: {integrity: sha512-Q4+qBFnN4bwGwvtXXzbp4P/4iNk0MaiGAzvQ8OiMtlLjkIKjmNN689uVzShSM0908q7GoFHXIPx4zi75ocoaHw==} - dev: true /copy-to-clipboard@3.3.3: resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==} @@ -30789,7 +30799,6 @@ packages: /docker-compose@0.22.2: resolution: {integrity: sha512-iXWb5+LiYmylIMFXvGTYsjI1F+Xyx78Jm/uj1dxwwZLbWkUdH6yOXY5Nr3RjbYX15EgbGJCq78d29CmWQQQMPg==} engines: {node: '>= 6.0.0'} - dev: true /doctrine@2.1.0: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} @@ -34822,7 +34831,6 @@ packages: lowercase-keys: 2.0.0 p-cancelable: 2.1.1 responselike: 2.0.1 - dev: true /got@12.6.1: resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} @@ -35780,7 +35788,6 @@ packages: dependencies: quick-lru: 5.1.1 resolve-alpn: 1.2.1 - dev: true /http2-wrapper@2.2.1: resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==} @@ -40636,7 +40643,6 @@ packages: engines: {node: '>=8'} dependencies: chalk: 2.4.2 - dev: true /log-symbols@4.1.0: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} @@ -40713,7 +40719,6 @@ packages: /lowercase-keys@2.0.0: resolution: {integrity: sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==} engines: {node: '>=8'} - dev: true /lowercase-keys@3.0.0: resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} @@ -41699,7 +41704,6 @@ packages: /mimic-response@1.0.1: resolution: {integrity: sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==} engines: {node: '>=4'} - dev: true /mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} @@ -42508,7 +42512,6 @@ packages: /normalize-url@6.1.0: resolution: {integrity: sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==} engines: {node: '>=10'} - dev: true /normalize-url@8.0.0: resolution: {integrity: sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==} @@ -43093,7 +43096,6 @@ packages: mute-stream: 0.0.8 strip-ansi: 6.0.1 wcwidth: 1.0.1 - dev: true /ora@5.4.1: resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==} @@ -43145,7 +43147,6 @@ packages: /p-cancelable@2.1.1: resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==} engines: {node: '>=8'} - dev: true /p-cancelable@3.0.0: resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} @@ -47616,7 +47617,6 @@ packages: resolution: {integrity: sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw==} dependencies: lowercase-keys: 2.0.0 - dev: true /responselike@3.0.0: resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==} @@ -48411,6 +48411,16 @@ packages: transitivePeerDependencies: - supports-color + /simple-git@3.5.0: + resolution: {integrity: sha512-fZsaq5nzdxQRhMNs6ESGLpMUHoL5GRP+boWPhq9pMYMKwOGZV2jHOxi8AbFFA2Y/6u4kR99HoULizSbpzaODkA==} + dependencies: + '@kwsites/file-exists': 1.1.1 + '@kwsites/promise-deferred': 1.1.1 + debug: 4.3.4(supports-color@9.4.0) + transitivePeerDependencies: + - supports-color + dev: false + /simple-html-tokenizer@0.5.11: resolution: {integrity: sha512-C2WEK/Z3HoSFbYq8tI7ni3eOo/NneSPRoPpcM7WdLjFOArFuyXEjAoCdOC3DgMfRyziZQ1hCNR4mrNdWEvD0og==} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c5eda4fbc84..455ddcf730c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -4,6 +4,7 @@ packages: - 'plugins/woocommerce/client/legacy' - 'tools/monorepo-merge' - 'tools/code-analyzer' + - 'tools/compare-perf' - 'tools/create-extension' - 'tools/package-release' - 'tools/cherry-pick' diff --git a/tools/compare-perf/config.js b/tools/compare-perf/config.js new file mode 100644 index 00000000000..b37724e5cc7 --- /dev/null +++ b/tools/compare-perf/config.js @@ -0,0 +1,11 @@ +const config = { + gitRepositoryURL: 'https://github.com/woocommerce/woocommerce.git', + setupCommand: + 'npm install -g pnpm && pnpm install &> /dev/null && pnpm build &> /dev/null && cd plugins/woocommerce && pnpm exec playwright install chromium', + pluginPath: '/plugins/woocommerce', + testsPath: '/plugins/woocommerce/tests/metrics/specs', + testCommand: + 'npm install -g pnpm && cd plugins/woocommerce && pnpm test:metrics', +}; + +module.exports = config; diff --git a/tools/compare-perf/index.js b/tools/compare-perf/index.js new file mode 100755 index 00000000000..d14680f9bf0 --- /dev/null +++ b/tools/compare-perf/index.js @@ -0,0 +1,41 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ + +const program = require( 'commander' ); +const { runPerformanceTests } = require( './performance' ); + +const catchException = ( command ) => { + return async ( ...args ) => { + try { + await command( ...args ); + } catch ( error ) { + console.error( error ); + process.exitCode = 1; + } + }; +}; + +const ciOption = [ '-c, --ci', 'Run in CI (non interactive)' ]; + +program + .command( 'compare-performance [branches...]' ) + .alias( 'perf' ) + .option( ...ciOption ) + .option( + '--rounds ', + 'Run each test suite this many times for each branch; results are summarized, default = 1' + ) + .option( + '--tests-branch ', + "Use this branch's performance test files" + ) + .option( + '--wp-version ', + 'Specify a WordPress version on which to test all branches' + ) + .description( + 'Runs performance tests on two separate branches and outputs the result' + ) + .action( catchException( runPerformanceTests ) ); + +program.parse( process.argv ); diff --git a/tools/compare-perf/package.json b/tools/compare-perf/package.json new file mode 100644 index 00000000000..334830ae7a6 --- /dev/null +++ b/tools/compare-perf/package.json @@ -0,0 +1,23 @@ +{ + "name": "compare-perf", + "version": "0.0.1", + "description": "A tool to compare performance accross tow branches in WooCommerce Monorepo.", + "author": "Automattic", + "homepage": "https://github.com/woocommerce/woocommerce", + "license": "GPLv2", + "repository": "woocommerce/woocommerce", + "scripts": { + "test": "node index.js" + }, + "dependencies": { + "@wordpress/env": "^8.13.0", + "commander": "9.5.0", + "chalk": "^4.1.2", + "inquirer": "^7.1.0", + "simple-git": "3.5.0" + }, + "engines": { + "node": "^16.14.1", + "pnpm": "^8.6.7" + } +} diff --git a/tools/compare-perf/performance.js b/tools/compare-perf/performance.js new file mode 100644 index 00000000000..66cc13c79b0 --- /dev/null +++ b/tools/compare-perf/performance.js @@ -0,0 +1,429 @@ +/* eslint-disable no-console */ +const os = require( 'os' ); +const fs = require( 'fs' ); +const path = require( 'path' ); +const SimpleGit = require( 'simple-git' ); +const chalk = require( 'chalk' ); + +const formats = { + title: chalk.bold, + error: chalk.bold.red, + warning: chalk.bold.keyword( 'orange' ), + success: chalk.bold.green, +}; +const { + runShellScript, + readJSONFile, + askForConfirmation, + getFilesFromDir, +} = require( './utils' ); +const config = require( './config' ); + +const ARTIFACTS_PATH = + process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' ); +const RESULTS_FILE_SUFFIX = '.performance-results.json'; + +/** + * @typedef WPPerformanceCommandOptions + * + * @property {boolean=} ci Run on CI. + * @property {number=} rounds Run each test suite this many times for each branch. + * @property {string=} testsBranch The branch whose performance test files will be used for testing. + * @property {string=} wpVersion The WordPress version to be used as the base install for testing. + */ + +/** + * A logging helper for printing steps and their substeps. + * + * @param {number} indent Value to indent the log. + * @param {any} msg Message to log. + * @param {...any} args Rest of the arguments to pass to console.log. + */ +function logAtIndent( indent, msg, ...args ) { + const prefix = indent === 0 ? 'ā–¶ ' : '> '; + const newline = indent === 0 ? '\n' : ''; + return console.log( + newline + ' '.repeat( indent ) + prefix + msg, + ...args + ); +} + +/** + * Sanitizes branch name to be used in a path or a filename. + * + * @param {string} branch + * + * @return {string} Sanitized branch name. + */ +function sanitizeBranchName( branch ) { + return branch.replace( /[^a-zA-Z0-9-]/g, '-' ); +} + +/** + * Computes the median number from an array numbers. + * + * @param {number[]} array + * + * @return {number|undefined} Median value or undefined if array empty. + */ +function median( array ) { + if ( ! array || ! array.length ) return undefined; + + const numbers = [ ...array ].sort( ( a, b ) => a - b ); + const middleIndex = Math.floor( numbers.length / 2 ); + + if ( numbers.length % 2 === 0 ) { + return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2; + } + return numbers[ middleIndex ]; +} + +/** + * Runs the performance tests on the current branch. + * + * @param {string} testSuite Name of the tests set. + * @param {string} testRunnerDir Path to the performance tests' clone. + * @param {string} runKey Unique identifier for the test run. + */ +async function runTestSuite( testSuite, testRunnerDir, runKey ) { + await runShellScript( + `${ config.testCommand } ${ testSuite }`, + testRunnerDir, + { + ...process.env, + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1', + WP_ARTIFACTS_PATH: ARTIFACTS_PATH, + RESULTS_ID: runKey, + } + ); +} + +/** + * Runs the performances tests on an array of branches and output the result. + * + * @param {string[]} branches Branches to compare + * @param {WPPerformanceCommandOptions} options Command options. + */ +async function runPerformanceTests( branches, options ) { + const runningInCI = !! process.env.CI || !! options.ci; + const TEST_ROUNDS = options.rounds || 1; + + // The default value doesn't work because commander provides an array. + if ( branches.length === 0 ) { + branches = [ 'trunk' ]; + } + + console.log( formats.title( '\nšŸ’ƒ Performance Tests šŸ•ŗ' ) ); + console.log( + '\nWelcome! This tool runs the performance tests on multiple branches and displays a comparison table.' + ); + + if ( ! runningInCI ) { + console.log( + formats.warning( + '\nIn order to run the tests, the tool is going to load a WordPress testing environment.' + + '\nMake sure to disable your own environment and testing ports before continuing.\n' + ) + ); + + await askForConfirmation( 'Ready to go? ' ); + } + + logAtIndent( 0, 'Setting up' ); + + /** + * @type {string[]} git refs against which to run tests; + * could be commit SHA, branch name, tag, etc... + */ + if ( branches.length < 2 ) { + throw new Error( `Need at least two git refs to run` ); + } + + const baseDir = path.join( os.tmpdir(), 'wp-performance-tests' ); + + if ( fs.existsSync( baseDir ) ) { + logAtIndent( 1, 'Removing existing files' ); + fs.rmSync( baseDir, { recursive: true } ); + } + + logAtIndent( 1, 'Creating base directory:', formats.success( baseDir ) ); + fs.mkdirSync( baseDir ); + + logAtIndent( 1, 'Setting up repository' ); + const sourceDir = path.join( baseDir, 'source' ); + + logAtIndent( 2, 'Creating directory:', formats.success( sourceDir ) ); + fs.mkdirSync( sourceDir ); + + const sourceGit = SimpleGit( sourceDir ); + logAtIndent( + 2, + 'Initializing:', + formats.success( config.gitRepositoryURL ) + ); + await sourceGit + .raw( 'init' ) + .raw( 'remote', 'add', 'origin', config.gitRepositoryURL ); + + for ( const [ i, branch ] of branches.entries() ) { + logAtIndent( + 2, + `Fetching environment branch (${ i + 1 } of ${ branches.length }):`, + formats.success( branch ) + ); + await sourceGit.raw( 'fetch', '--depth=1', 'origin', branch ); + } + + const testRunnerBranch = options.testsBranch || branches[ 0 ]; + if ( options.testsBranch && ! branches.includes( options.testsBranch ) ) { + logAtIndent( + 2, + 'Fetching test runner branch:', + formats.success( options.testsBranch ) + ); + await sourceGit.raw( + 'fetch', + '--depth=1', + 'origin', + options.testsBranch + ); + } else { + logAtIndent( + 2, + 'Using test runner branch:', + formats.success( testRunnerBranch ) + ); + } + + logAtIndent( 1, 'Setting up test runner' ); + + const testRunnerDir = path.join( baseDir + '/tests' ); + + logAtIndent( 2, 'Copying source to:', formats.success( testRunnerDir ) ); + await runShellScript( `cp -R ${ sourceDir } ${ testRunnerDir }` ); + + logAtIndent( + 2, + 'Checking out branch:', + formats.success( testRunnerBranch ) + ); + await SimpleGit( testRunnerDir ).raw( 'checkout', testRunnerBranch ); + + logAtIndent( 2, 'Installing dependencies and building' ); + await runShellScript( + `bash -c "source $HOME/.nvm/nvm.sh && nvm install && ${ config.setupCommand }"`, + testRunnerDir + ); + + logAtIndent( 1, 'Setting up test environments' ); + + const envsDir = path.join( baseDir, 'environments' ); + logAtIndent( 2, 'Creating parent directory:', formats.success( envsDir ) ); + fs.mkdirSync( envsDir ); + + let wpZipUrl = null; + if ( options.wpVersion ) { + // In order to match the topology of ZIP files at wp.org, remap .0 + // patch versions to major versions: + // + // 5.7 -> 5.7 (unchanged) + // 5.7.0 -> 5.7 (changed) + // 5.7.2 -> 5.7.2 (unchanged) + const zipVersion = options.wpVersion.replace( /^(\d+\.\d+).0/, '$1' ); + wpZipUrl = `https://wordpress.org/wordpress-${ zipVersion }.zip`; + } + + const branchDirs = {}; + for ( const branch of branches ) { + logAtIndent( 2, 'Branch:', formats.success( branch ) ); + const sanitizedBranchName = sanitizeBranchName( branch ); + const envDir = path.join( envsDir, sanitizedBranchName ); + + logAtIndent( 3, 'Creating directory:', formats.success( envDir ) ); + fs.mkdirSync( envDir ); + branchDirs[ branch ] = envDir; + const buildDir = path.join( envDir, 'plugin' ); + + logAtIndent( 3, 'Copying source to:', formats.success( buildDir ) ); + await runShellScript( `cp -R ${ sourceDir } ${ buildDir }` ); + + logAtIndent( 3, 'Checking out:', formats.success( branch ) ); + await SimpleGit( buildDir ).raw( 'checkout', branch ); + + logAtIndent( 3, 'Installing dependencies and building' ); + await runShellScript( + `bash -c "source $HOME/.nvm/nvm.sh && nvm install && ${ config.setupCommand }"`, + buildDir + ); + + const wpEnvConfigPath = path.join( envDir, '.wp-env.json' ); + + logAtIndent( + 3, + 'Saving wp-env config to:', + formats.success( wpEnvConfigPath ) + ); + + fs.writeFileSync( + wpEnvConfigPath, + JSON.stringify( + { + config: { + WP_DEBUG: false, + SCRIPT_DEBUG: false, + }, + core: wpZipUrl || 'WordPress/WordPress', + plugins: [ buildDir + config.pluginPath ], + themes: [ + // Ideally this should be a fixed version of the theme. + // And it should be enabled in the tests suite. + 'https://downloads.wordpress.org/theme/twentynineteen.zip', + ], + env: { + tests: { + port: 8086, + }, + }, + }, + null, + 2 + ), + 'utf8' + ); + } + + logAtIndent( 0, 'Looking for test files' ); + + const testSuites = getFilesFromDir( + path.join( testRunnerDir, config.testsPath ) + ).map( ( file ) => { + logAtIndent( 1, 'Found:', formats.success( file ) ); + return path.basename( file, '.spec.js' ); + } ); + + logAtIndent( 0, 'Running tests' ); + + if ( wpZipUrl ) { + logAtIndent( + 1, + 'Using:', + formats.success( `WordPress v${ options.wpVersion }` ) + ); + } else { + logAtIndent( 1, 'Using:', formats.success( 'WordPress trunk' ) ); + } + + // TODO: change this to tools/compare-perf/node_modules/.bin/wp-env + const wpEnvPath = path.join( + testRunnerDir, + 'plugins/woocommerce/node_modules/.bin/wp-env' + ); + + for ( const testSuite of testSuites ) { + for ( let i = 1; i <= TEST_ROUNDS; i++ ) { + logAtIndent( + 1, + // prettier-ignore + `Suite: ${ formats.success( testSuite ) } (round ${ i } of ${ TEST_ROUNDS })` + ); + + for ( const branch of branches ) { + logAtIndent( 2, 'Branch:', formats.success( branch ) ); + + const sanitizedBranchName = sanitizeBranchName( branch ); + const runKey = `${ testSuite }_${ sanitizedBranchName }_round-${ i }`; + const envDir = branchDirs[ branch ]; + + logAtIndent( 3, 'Starting environment' ); + await runShellScript( `${ wpEnvPath } start`, envDir ); + + logAtIndent( 3, 'Running tests' ); + await runTestSuite( testSuite, testRunnerDir, runKey ); + + logAtIndent( 3, 'Stopping environment' ); + await runShellScript( `${ wpEnvPath } stop`, envDir ); + } + } + } + + logAtIndent( 0, 'Calculating results' ); + + const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) => + file.endsWith( RESULTS_FILE_SUFFIX ) + ); + /** @type {Record>>} */ + const results = {}; + + // Calculate medians from all rounds. + for ( const testSuite of testSuites ) { + logAtIndent( 1, 'Test suite:', formats.success( testSuite ) ); + + results[ testSuite ] = {}; + for ( const branch of branches ) { + const sanitizedBranchName = sanitizeBranchName( branch ); + const resultsRounds = resultFiles + .filter( ( file ) => + file.includes( + `${ testSuite }_${ sanitizedBranchName }_round-` + ) + ) + .map( ( file ) => { + logAtIndent( 2, 'Reading from:', formats.success( file ) ); + return readJSONFile( file ); + } ); + + const metrics = Object.keys( resultsRounds[ 0 ] ); + results[ testSuite ][ branch ] = {}; + + for ( const metric of metrics ) { + const values = resultsRounds + .map( ( round ) => round[ metric ] ) + .filter( ( value ) => typeof value === 'number' ); + + const value = median( values ); + if ( value !== undefined ) { + results[ testSuite ][ branch ][ metric ] = value; + } + } + } + const calculatedResultsPath = path.join( + ARTIFACTS_PATH, + testSuite + RESULTS_FILE_SUFFIX + ); + + logAtIndent( + 2, + 'Saving curated results to:', + formats.success( calculatedResultsPath ) + ); + fs.writeFileSync( + calculatedResultsPath, + JSON.stringify( results[ testSuite ], null, 2 ) + ); + } + + logAtIndent( 0, 'Printing results' ); + + for ( const testSuite of testSuites ) { + logAtIndent( 0, formats.success( testSuite ) ); + + // Invert the results so we can display them in a table. + /** @type {Record>} */ + const invertedResult = {}; + for ( const [ branch, metrics ] of Object.entries( + results[ testSuite ] + ) ) { + for ( const [ metric, value ] of Object.entries( metrics ) ) { + invertedResult[ metric ] = invertedResult[ metric ] || {}; + invertedResult[ metric ][ branch ] = `${ value } ms`; + } + } + + // Print the results. + console.table( invertedResult ); + } +} + +module.exports = { + runPerformanceTests, +}; diff --git a/tools/compare-perf/utils.js b/tools/compare-perf/utils.js new file mode 100644 index 00000000000..2cca3b1a781 --- /dev/null +++ b/tools/compare-perf/utils.js @@ -0,0 +1,105 @@ +/* eslint-disable no-console */ +const inquirer = require( 'inquirer' ); +const fs = require( 'fs' ); +const childProcess = require( 'child_process' ); +const path = require( 'path' ); +const chalk = require( 'chalk' ); + +/** + * Utility to run a child script + * + * @typedef {NodeJS.ProcessEnv} Env + * + * @param {string} script Script to run. + * @param {string=} cwd Working directory. + * @param {Env=} env Additional environment variables to pass to the script. + */ +function runShellScript( script, cwd, env = {} ) { + return new Promise( ( resolve, reject ) => { + childProcess.exec( + script, + { + cwd, + env: { + NO_CHECKS: 'true', + PATH: process.env.PATH, + HOME: process.env.HOME, + USER: process.env.USER, + ...env, + }, + }, + function ( error, stdout, stderr ) { + if ( error ) { + console.log( stdout ); // Sometimes the error message is thrown via stdout. + console.log( stderr ); + reject( error ); + } else { + resolve( true ); + } + } + ); + } ); +} + +/** + * Small utility used to read an uncached version of a JSON file + * + * @param {string} fileName + */ +function readJSONFile( fileName ) { + const data = fs.readFileSync( fileName, 'utf8' ); + return JSON.parse( data ); +} + +/** + * Asks the user for a confirmation to continue or abort otherwise. + * + * @param {string} message Confirmation message. + * @param {boolean} isDefault Default reply. + * @param {string} abortMessage Abort message. + */ +async function askForConfirmation( + message, + isDefault = true, + abortMessage = 'Aborting.' +) { + const { isReady } = await inquirer.prompt( [ + { + type: 'confirm', + name: 'isReady', + default: isDefault, + message, + }, + ] ); + + if ( ! isReady ) { + chalk.log( chalk.bold.red( '\n' + abortMessage ) ); + process.exit( 1 ); + } +} + +/** + * Scans the given directory and returns an array of file paths. + * + * @param {string} dir The path to the directory to scan. + * + * @return {string[]} An array of file paths. + */ +function getFilesFromDir( dir ) { + if ( ! fs.existsSync( dir ) ) { + console.log( 'Directory does not exist: ', dir ); + return []; + } + + return fs + .readdirSync( dir, { withFileTypes: true } ) + .filter( ( dirent ) => dirent.isFile() ) + .map( ( dirent ) => path.join( dir, dirent.name ) ); +} + +module.exports = { + askForConfirmation, + readJSONFile, + runShellScript, + getFilesFromDir, +};