Performance: Add a tool and a CI workflow to compare performance between PR and trunk and track metrics on trunk
This commit is contained in:
Riad Benguella
parent
2b54c8a61d
commit
68617e8ef1
|
|
@ -0,0 +1,53 @@
|
|||
name: Metrics Tracking
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [trunk]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
metrics:
|
||||
name: Run metrics tests
|
||||
runs-on: ubuntu-20.04
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
WP_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Setup WooCommerce Monorepo
|
||||
uses: ./.github/actions/setup-woocommerce-monorepo
|
||||
with:
|
||||
install: '@woocommerce/plugin-woocommerce...'
|
||||
build: '@woocommerce/plugin-woocommerce'
|
||||
|
||||
- name: Compare performance with trunk
|
||||
if: github.event_name == 'pull_request'
|
||||
run: cd tools/compare-perf && pnpm run perf $GITHUB_SHA trunk --tests-branch $GITHUB_SHA
|
||||
|
||||
- name: Compare performance with base branch
|
||||
if: github.event_name == 'push'
|
||||
# The base hash used here need to be a commit that is compatible with the current WP version
|
||||
# The current one is 2f6ca66e00b3666b2567877ae67cbfb5b6ce171a
|
||||
# it needs to be updated every time it becomes unsupported by the current wp-env (WP version).
|
||||
# It is used as a base comparison point to avoid fluctuation in the performance metrics.
|
||||
run: |
|
||||
WP_VERSION=$(awk -F ': ' '/^Tested up to/{print $2}' readme.txt)
|
||||
IFS=. read -ra WP_VERSION_ARRAY <<< "$WP_VERSION"
|
||||
WP_MAJOR="${WP_VERSION_ARRAY[0]}.${WP_VERSION_ARRAY[1]}"
|
||||
cd tools/compare-perf && pnpm run perf $GITHUB_SHA 2f6ca66e00b3666b2567877ae67cbfb5b6ce171a --tests-branch $GITHUB_SHA --wp-version "$WP_MAJOR"
|
||||
|
||||
- name: Archive performance results
|
||||
if: success()
|
||||
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
|
||||
with:
|
||||
name: performance-results
|
||||
path: ${{ env.WP_ARTIFACTS_PATH }}/*.performance-results*.json
|
||||
|
|
@ -108,3 +108,4 @@ changes.json
|
|||
# Metrics tests
|
||||
/artifacts
|
||||
/plugins/*/artifacts
|
||||
/tools/*/artifacts
|
||||
|
|
|
|||
|
|
@ -4610,6 +4610,24 @@ importers:
|
|||
specifier: 0.14.1
|
||||
version: 0.14.1
|
||||
|
||||
tools/compare-perf:
|
||||
dependencies:
|
||||
'@wordpress/env':
|
||||
specifier: ^8.13.0
|
||||
version: 8.13.0
|
||||
chalk:
|
||||
specifier: ^4.1.2
|
||||
version: 4.1.2
|
||||
commander:
|
||||
specifier: 9.5.0
|
||||
version: 9.5.0
|
||||
inquirer:
|
||||
specifier: ^7.1.0
|
||||
version: 7.3.3
|
||||
simple-git:
|
||||
specifier: 3.5.0
|
||||
version: 3.5.0
|
||||
|
||||
tools/create-extension:
|
||||
dependencies:
|
||||
chalk:
|
||||
|
|
@ -14670,7 +14688,6 @@ packages:
|
|||
/@sindresorhus/is@4.6.0:
|
||||
resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==}
|
||||
engines: {node: '>=10'}
|
||||
dev: true
|
||||
|
||||
/@sindresorhus/is@5.6.0:
|
||||
resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==}
|
||||
|
|
@ -18448,7 +18465,6 @@ packages:
|
|||
engines: {node: '>=10'}
|
||||
dependencies:
|
||||
defer-to-connect: 2.0.1
|
||||
dev: true
|
||||
|
||||
/@szmarczak/http-timer@5.0.1:
|
||||
resolution: {integrity: sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==}
|
||||
|
|
@ -18744,7 +18760,6 @@ packages:
|
|||
'@types/keyv': 3.1.4
|
||||
'@types/node': 20.10.4
|
||||
'@types/responselike': 1.0.3
|
||||
dev: true
|
||||
|
||||
/@types/cheerio@0.22.35:
|
||||
resolution: {integrity: sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==}
|
||||
|
|
@ -23250,7 +23265,6 @@ packages:
|
|||
yargs: 17.7.2
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
dev: true
|
||||
|
||||
/@wordpress/env@8.2.0:
|
||||
resolution: {integrity: sha512-MGf2TJD6MbBjDn/+feGAcXnh6ct9Y/aCjIZTtbMI+b+pXAPh0RzR7Q5D6KMgKUNcGeL6nNQmEUUSSPuzV2UWfQ==}
|
||||
|
|
@ -27864,7 +27878,7 @@ packages:
|
|||
resolution: {integrity: sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==}
|
||||
dependencies:
|
||||
base64-js: 1.5.1
|
||||
ieee754: 1.1.13
|
||||
ieee754: 1.2.1
|
||||
isarray: 1.0.0
|
||||
dev: true
|
||||
|
||||
|
|
@ -28047,7 +28061,6 @@ packages:
|
|||
/cacheable-lookup@5.0.4:
|
||||
resolution: {integrity: sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==}
|
||||
engines: {node: '>=10.6.0'}
|
||||
dev: true
|
||||
|
||||
/cacheable-lookup@7.0.0:
|
||||
resolution: {integrity: sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==}
|
||||
|
|
@ -28078,7 +28091,6 @@ packages:
|
|||
lowercase-keys: 2.0.0
|
||||
normalize-url: 6.1.0
|
||||
responselike: 2.0.1
|
||||
dev: true
|
||||
|
||||
/call-bind@1.0.5:
|
||||
resolution: {integrity: sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==}
|
||||
|
|
@ -28772,7 +28784,6 @@ packages:
|
|||
resolution: {integrity: sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==}
|
||||
dependencies:
|
||||
mimic-response: 1.0.1
|
||||
dev: true
|
||||
|
||||
/clone-stats@1.0.0:
|
||||
resolution: {integrity: sha512-au6ydSpg6nsrigcZ4m8Bc9hxjeW+GJ8xh5G3BJCMt4WXe1H10UNaVOamqQTmrx1kjVuxAHIQSNU6hY4Nsn9/ag==}
|
||||
|
|
@ -29269,7 +29280,6 @@ packages:
|
|||
|
||||
/copy-dir@1.3.0:
|
||||
resolution: {integrity: sha512-Q4+qBFnN4bwGwvtXXzbp4P/4iNk0MaiGAzvQ8OiMtlLjkIKjmNN689uVzShSM0908q7GoFHXIPx4zi75ocoaHw==}
|
||||
dev: true
|
||||
|
||||
/copy-to-clipboard@3.3.3:
|
||||
resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==}
|
||||
|
|
@ -30789,7 +30799,6 @@ packages:
|
|||
/docker-compose@0.22.2:
|
||||
resolution: {integrity: sha512-iXWb5+LiYmylIMFXvGTYsjI1F+Xyx78Jm/uj1dxwwZLbWkUdH6yOXY5Nr3RjbYX15EgbGJCq78d29CmWQQQMPg==}
|
||||
engines: {node: '>= 6.0.0'}
|
||||
dev: true
|
||||
|
||||
/doctrine@2.1.0:
|
||||
resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==}
|
||||
|
|
@ -34822,7 +34831,6 @@ packages:
|
|||
lowercase-keys: 2.0.0
|
||||
p-cancelable: 2.1.1
|
||||
responselike: 2.0.1
|
||||
dev: true
|
||||
|
||||
/got@12.6.1:
|
||||
resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==}
|
||||
|
|
@ -35780,7 +35788,6 @@ packages:
|
|||
dependencies:
|
||||
quick-lru: 5.1.1
|
||||
resolve-alpn: 1.2.1
|
||||
dev: true
|
||||
|
||||
/http2-wrapper@2.2.1:
|
||||
resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==}
|
||||
|
|
@ -40636,7 +40643,6 @@ packages:
|
|||
engines: {node: '>=8'}
|
||||
dependencies:
|
||||
chalk: 2.4.2
|
||||
dev: true
|
||||
|
||||
/log-symbols@4.1.0:
|
||||
resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==}
|
||||
|
|
@ -40713,7 +40719,6 @@ packages:
|
|||
/lowercase-keys@2.0.0:
|
||||
resolution: {integrity: sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==}
|
||||
engines: {node: '>=8'}
|
||||
dev: true
|
||||
|
||||
/lowercase-keys@3.0.0:
|
||||
resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==}
|
||||
|
|
@ -41699,7 +41704,6 @@ packages:
|
|||
/mimic-response@1.0.1:
|
||||
resolution: {integrity: sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==}
|
||||
engines: {node: '>=4'}
|
||||
dev: true
|
||||
|
||||
/mimic-response@3.1.0:
|
||||
resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
|
||||
|
|
@ -42508,7 +42512,6 @@ packages:
|
|||
/normalize-url@6.1.0:
|
||||
resolution: {integrity: sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==}
|
||||
engines: {node: '>=10'}
|
||||
dev: true
|
||||
|
||||
/normalize-url@8.0.0:
|
||||
resolution: {integrity: sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==}
|
||||
|
|
@ -43093,7 +43096,6 @@ packages:
|
|||
mute-stream: 0.0.8
|
||||
strip-ansi: 6.0.1
|
||||
wcwidth: 1.0.1
|
||||
dev: true
|
||||
|
||||
/ora@5.4.1:
|
||||
resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==}
|
||||
|
|
@ -43145,7 +43147,6 @@ packages:
|
|||
/p-cancelable@2.1.1:
|
||||
resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==}
|
||||
engines: {node: '>=8'}
|
||||
dev: true
|
||||
|
||||
/p-cancelable@3.0.0:
|
||||
resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==}
|
||||
|
|
@ -47616,7 +47617,6 @@ packages:
|
|||
resolution: {integrity: sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw==}
|
||||
dependencies:
|
||||
lowercase-keys: 2.0.0
|
||||
dev: true
|
||||
|
||||
/responselike@3.0.0:
|
||||
resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==}
|
||||
|
|
@ -48411,6 +48411,16 @@ packages:
|
|||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
/simple-git@3.5.0:
|
||||
resolution: {integrity: sha512-fZsaq5nzdxQRhMNs6ESGLpMUHoL5GRP+boWPhq9pMYMKwOGZV2jHOxi8AbFFA2Y/6u4kR99HoULizSbpzaODkA==}
|
||||
dependencies:
|
||||
'@kwsites/file-exists': 1.1.1
|
||||
'@kwsites/promise-deferred': 1.1.1
|
||||
debug: 4.3.4(supports-color@9.4.0)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
dev: false
|
||||
|
||||
/simple-html-tokenizer@0.5.11:
|
||||
resolution: {integrity: sha512-C2WEK/Z3HoSFbYq8tI7ni3eOo/NneSPRoPpcM7WdLjFOArFuyXEjAoCdOC3DgMfRyziZQ1hCNR4mrNdWEvD0og==}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ packages:
|
|||
- 'plugins/woocommerce/client/legacy'
|
||||
- 'tools/monorepo-merge'
|
||||
- 'tools/code-analyzer'
|
||||
- 'tools/compare-perf'
|
||||
- 'tools/create-extension'
|
||||
- 'tools/package-release'
|
||||
- 'tools/cherry-pick'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
const config = {
|
||||
gitRepositoryURL: 'https://github.com/woocommerce/woocommerce.git',
|
||||
setupCommand:
|
||||
'npm install -g pnpm && pnpm install &> /dev/null && pnpm build &> /dev/null && cd plugins/woocommerce && pnpm exec playwright install chromium',
|
||||
pluginPath: '/plugins/woocommerce',
|
||||
testsPath: '/plugins/woocommerce/tests/metrics/specs',
|
||||
testCommand:
|
||||
'npm install -g pnpm && cd plugins/woocommerce && pnpm test:metrics',
|
||||
};
|
||||
|
||||
module.exports = config;
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
#!/usr/bin/env node
|
||||
/* eslint-disable no-console */
|
||||
|
||||
const program = require( 'commander' );
|
||||
const { runPerformanceTests } = require( './performance' );
|
||||
|
||||
const catchException = ( command ) => {
|
||||
return async ( ...args ) => {
|
||||
try {
|
||||
await command( ...args );
|
||||
} catch ( error ) {
|
||||
console.error( error );
|
||||
process.exitCode = 1;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
const ciOption = [ '-c, --ci', 'Run in CI (non interactive)' ];
|
||||
|
||||
program
|
||||
.command( 'compare-performance [branches...]' )
|
||||
.alias( 'perf' )
|
||||
.option( ...ciOption )
|
||||
.option(
|
||||
'--rounds <count>',
|
||||
'Run each test suite this many times for each branch; results are summarized, default = 1'
|
||||
)
|
||||
.option(
|
||||
'--tests-branch <branch>',
|
||||
"Use this branch's performance test files"
|
||||
)
|
||||
.option(
|
||||
'--wp-version <version>',
|
||||
'Specify a WordPress version on which to test all branches'
|
||||
)
|
||||
.description(
|
||||
'Runs performance tests on two separate branches and outputs the result'
|
||||
)
|
||||
.action( catchException( runPerformanceTests ) );
|
||||
|
||||
program.parse( process.argv );
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
"name": "compare-perf",
|
||||
"version": "0.0.1",
|
||||
"description": "A tool to compare performance accross tow branches in WooCommerce Monorepo.",
|
||||
"author": "Automattic",
|
||||
"homepage": "https://github.com/woocommerce/woocommerce",
|
||||
"license": "GPLv2",
|
||||
"repository": "woocommerce/woocommerce",
|
||||
"scripts": {
|
||||
"test": "node index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@wordpress/env": "^8.13.0",
|
||||
"commander": "9.5.0",
|
||||
"chalk": "^4.1.2",
|
||||
"inquirer": "^7.1.0",
|
||||
"simple-git": "3.5.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^16.14.1",
|
||||
"pnpm": "^8.6.7"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,429 @@
|
|||
/* eslint-disable no-console */
|
||||
const os = require( 'os' );
|
||||
const fs = require( 'fs' );
|
||||
const path = require( 'path' );
|
||||
const SimpleGit = require( 'simple-git' );
|
||||
const chalk = require( 'chalk' );
|
||||
|
||||
const formats = {
|
||||
title: chalk.bold,
|
||||
error: chalk.bold.red,
|
||||
warning: chalk.bold.keyword( 'orange' ),
|
||||
success: chalk.bold.green,
|
||||
};
|
||||
const {
|
||||
runShellScript,
|
||||
readJSONFile,
|
||||
askForConfirmation,
|
||||
getFilesFromDir,
|
||||
} = require( './utils' );
|
||||
const config = require( './config' );
|
||||
|
||||
const ARTIFACTS_PATH =
|
||||
process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
|
||||
const RESULTS_FILE_SUFFIX = '.performance-results.json';
|
||||
|
||||
/**
|
||||
* @typedef WPPerformanceCommandOptions
|
||||
*
|
||||
* @property {boolean=} ci Run on CI.
|
||||
* @property {number=} rounds Run each test suite this many times for each branch.
|
||||
* @property {string=} testsBranch The branch whose performance test files will be used for testing.
|
||||
* @property {string=} wpVersion The WordPress version to be used as the base install for testing.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A logging helper for printing steps and their substeps.
|
||||
*
|
||||
* @param {number} indent Value to indent the log.
|
||||
* @param {any} msg Message to log.
|
||||
* @param {...any} args Rest of the arguments to pass to console.log.
|
||||
*/
|
||||
function logAtIndent( indent, msg, ...args ) {
|
||||
const prefix = indent === 0 ? '▶ ' : '> ';
|
||||
const newline = indent === 0 ? '\n' : '';
|
||||
return console.log(
|
||||
newline + ' '.repeat( indent ) + prefix + msg,
|
||||
...args
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitizes branch name to be used in a path or a filename.
|
||||
*
|
||||
* @param {string} branch
|
||||
*
|
||||
* @return {string} Sanitized branch name.
|
||||
*/
|
||||
function sanitizeBranchName( branch ) {
|
||||
return branch.replace( /[^a-zA-Z0-9-]/g, '-' );
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the median number from an array numbers.
|
||||
*
|
||||
* @param {number[]} array
|
||||
*
|
||||
* @return {number|undefined} Median value or undefined if array empty.
|
||||
*/
|
||||
function median( array ) {
|
||||
if ( ! array || ! array.length ) return undefined;
|
||||
|
||||
const numbers = [ ...array ].sort( ( a, b ) => a - b );
|
||||
const middleIndex = Math.floor( numbers.length / 2 );
|
||||
|
||||
if ( numbers.length % 2 === 0 ) {
|
||||
return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
|
||||
}
|
||||
return numbers[ middleIndex ];
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the performance tests on the current branch.
|
||||
*
|
||||
* @param {string} testSuite Name of the tests set.
|
||||
* @param {string} testRunnerDir Path to the performance tests' clone.
|
||||
* @param {string} runKey Unique identifier for the test run.
|
||||
*/
|
||||
async function runTestSuite( testSuite, testRunnerDir, runKey ) {
|
||||
await runShellScript(
|
||||
`${ config.testCommand } ${ testSuite }`,
|
||||
testRunnerDir,
|
||||
{
|
||||
...process.env,
|
||||
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1',
|
||||
WP_ARTIFACTS_PATH: ARTIFACTS_PATH,
|
||||
RESULTS_ID: runKey,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the performances tests on an array of branches and output the result.
|
||||
*
|
||||
* @param {string[]} branches Branches to compare
|
||||
* @param {WPPerformanceCommandOptions} options Command options.
|
||||
*/
|
||||
async function runPerformanceTests( branches, options ) {
|
||||
const runningInCI = !! process.env.CI || !! options.ci;
|
||||
const TEST_ROUNDS = options.rounds || 1;
|
||||
|
||||
// The default value doesn't work because commander provides an array.
|
||||
if ( branches.length === 0 ) {
|
||||
branches = [ 'trunk' ];
|
||||
}
|
||||
|
||||
console.log( formats.title( '\n💃 Performance Tests 🕺' ) );
|
||||
console.log(
|
||||
'\nWelcome! This tool runs the performance tests on multiple branches and displays a comparison table.'
|
||||
);
|
||||
|
||||
if ( ! runningInCI ) {
|
||||
console.log(
|
||||
formats.warning(
|
||||
'\nIn order to run the tests, the tool is going to load a WordPress testing environment.' +
|
||||
'\nMake sure to disable your own environment and testing ports before continuing.\n'
|
||||
)
|
||||
);
|
||||
|
||||
await askForConfirmation( 'Ready to go? ' );
|
||||
}
|
||||
|
||||
logAtIndent( 0, 'Setting up' );
|
||||
|
||||
/**
|
||||
* @type {string[]} git refs against which to run tests;
|
||||
* could be commit SHA, branch name, tag, etc...
|
||||
*/
|
||||
if ( branches.length < 2 ) {
|
||||
throw new Error( `Need at least two git refs to run` );
|
||||
}
|
||||
|
||||
const baseDir = path.join( os.tmpdir(), 'wp-performance-tests' );
|
||||
|
||||
if ( fs.existsSync( baseDir ) ) {
|
||||
logAtIndent( 1, 'Removing existing files' );
|
||||
fs.rmSync( baseDir, { recursive: true } );
|
||||
}
|
||||
|
||||
logAtIndent( 1, 'Creating base directory:', formats.success( baseDir ) );
|
||||
fs.mkdirSync( baseDir );
|
||||
|
||||
logAtIndent( 1, 'Setting up repository' );
|
||||
const sourceDir = path.join( baseDir, 'source' );
|
||||
|
||||
logAtIndent( 2, 'Creating directory:', formats.success( sourceDir ) );
|
||||
fs.mkdirSync( sourceDir );
|
||||
|
||||
const sourceGit = SimpleGit( sourceDir );
|
||||
logAtIndent(
|
||||
2,
|
||||
'Initializing:',
|
||||
formats.success( config.gitRepositoryURL )
|
||||
);
|
||||
await sourceGit
|
||||
.raw( 'init' )
|
||||
.raw( 'remote', 'add', 'origin', config.gitRepositoryURL );
|
||||
|
||||
for ( const [ i, branch ] of branches.entries() ) {
|
||||
logAtIndent(
|
||||
2,
|
||||
`Fetching environment branch (${ i + 1 } of ${ branches.length }):`,
|
||||
formats.success( branch )
|
||||
);
|
||||
await sourceGit.raw( 'fetch', '--depth=1', 'origin', branch );
|
||||
}
|
||||
|
||||
const testRunnerBranch = options.testsBranch || branches[ 0 ];
|
||||
if ( options.testsBranch && ! branches.includes( options.testsBranch ) ) {
|
||||
logAtIndent(
|
||||
2,
|
||||
'Fetching test runner branch:',
|
||||
formats.success( options.testsBranch )
|
||||
);
|
||||
await sourceGit.raw(
|
||||
'fetch',
|
||||
'--depth=1',
|
||||
'origin',
|
||||
options.testsBranch
|
||||
);
|
||||
} else {
|
||||
logAtIndent(
|
||||
2,
|
||||
'Using test runner branch:',
|
||||
formats.success( testRunnerBranch )
|
||||
);
|
||||
}
|
||||
|
||||
logAtIndent( 1, 'Setting up test runner' );
|
||||
|
||||
const testRunnerDir = path.join( baseDir + '/tests' );
|
||||
|
||||
logAtIndent( 2, 'Copying source to:', formats.success( testRunnerDir ) );
|
||||
await runShellScript( `cp -R ${ sourceDir } ${ testRunnerDir }` );
|
||||
|
||||
logAtIndent(
|
||||
2,
|
||||
'Checking out branch:',
|
||||
formats.success( testRunnerBranch )
|
||||
);
|
||||
await SimpleGit( testRunnerDir ).raw( 'checkout', testRunnerBranch );
|
||||
|
||||
logAtIndent( 2, 'Installing dependencies and building' );
|
||||
await runShellScript(
|
||||
`bash -c "source $HOME/.nvm/nvm.sh && nvm install && ${ config.setupCommand }"`,
|
||||
testRunnerDir
|
||||
);
|
||||
|
||||
logAtIndent( 1, 'Setting up test environments' );
|
||||
|
||||
const envsDir = path.join( baseDir, 'environments' );
|
||||
logAtIndent( 2, 'Creating parent directory:', formats.success( envsDir ) );
|
||||
fs.mkdirSync( envsDir );
|
||||
|
||||
let wpZipUrl = null;
|
||||
if ( options.wpVersion ) {
|
||||
// In order to match the topology of ZIP files at wp.org, remap .0
|
||||
// patch versions to major versions:
|
||||
//
|
||||
// 5.7 -> 5.7 (unchanged)
|
||||
// 5.7.0 -> 5.7 (changed)
|
||||
// 5.7.2 -> 5.7.2 (unchanged)
|
||||
const zipVersion = options.wpVersion.replace( /^(\d+\.\d+).0/, '$1' );
|
||||
wpZipUrl = `https://wordpress.org/wordpress-${ zipVersion }.zip`;
|
||||
}
|
||||
|
||||
const branchDirs = {};
|
||||
for ( const branch of branches ) {
|
||||
logAtIndent( 2, 'Branch:', formats.success( branch ) );
|
||||
const sanitizedBranchName = sanitizeBranchName( branch );
|
||||
const envDir = path.join( envsDir, sanitizedBranchName );
|
||||
|
||||
logAtIndent( 3, 'Creating directory:', formats.success( envDir ) );
|
||||
fs.mkdirSync( envDir );
|
||||
branchDirs[ branch ] = envDir;
|
||||
const buildDir = path.join( envDir, 'plugin' );
|
||||
|
||||
logAtIndent( 3, 'Copying source to:', formats.success( buildDir ) );
|
||||
await runShellScript( `cp -R ${ sourceDir } ${ buildDir }` );
|
||||
|
||||
logAtIndent( 3, 'Checking out:', formats.success( branch ) );
|
||||
await SimpleGit( buildDir ).raw( 'checkout', branch );
|
||||
|
||||
logAtIndent( 3, 'Installing dependencies and building' );
|
||||
await runShellScript(
|
||||
`bash -c "source $HOME/.nvm/nvm.sh && nvm install && ${ config.setupCommand }"`,
|
||||
buildDir
|
||||
);
|
||||
|
||||
const wpEnvConfigPath = path.join( envDir, '.wp-env.json' );
|
||||
|
||||
logAtIndent(
|
||||
3,
|
||||
'Saving wp-env config to:',
|
||||
formats.success( wpEnvConfigPath )
|
||||
);
|
||||
|
||||
fs.writeFileSync(
|
||||
wpEnvConfigPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
config: {
|
||||
WP_DEBUG: false,
|
||||
SCRIPT_DEBUG: false,
|
||||
},
|
||||
core: wpZipUrl || 'WordPress/WordPress',
|
||||
plugins: [ buildDir + config.pluginPath ],
|
||||
themes: [
|
||||
// Ideally this should be a fixed version of the theme.
|
||||
// And it should be enabled in the tests suite.
|
||||
'https://downloads.wordpress.org/theme/twentynineteen.zip',
|
||||
],
|
||||
env: {
|
||||
tests: {
|
||||
port: 8086,
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2
|
||||
),
|
||||
'utf8'
|
||||
);
|
||||
}
|
||||
|
||||
logAtIndent( 0, 'Looking for test files' );
|
||||
|
||||
const testSuites = getFilesFromDir(
|
||||
path.join( testRunnerDir, config.testsPath )
|
||||
).map( ( file ) => {
|
||||
logAtIndent( 1, 'Found:', formats.success( file ) );
|
||||
return path.basename( file, '.spec.js' );
|
||||
} );
|
||||
|
||||
logAtIndent( 0, 'Running tests' );
|
||||
|
||||
if ( wpZipUrl ) {
|
||||
logAtIndent(
|
||||
1,
|
||||
'Using:',
|
||||
formats.success( `WordPress v${ options.wpVersion }` )
|
||||
);
|
||||
} else {
|
||||
logAtIndent( 1, 'Using:', formats.success( 'WordPress trunk' ) );
|
||||
}
|
||||
|
||||
// TODO: change this to tools/compare-perf/node_modules/.bin/wp-env
|
||||
const wpEnvPath = path.join(
|
||||
testRunnerDir,
|
||||
'plugins/woocommerce/node_modules/.bin/wp-env'
|
||||
);
|
||||
|
||||
for ( const testSuite of testSuites ) {
|
||||
for ( let i = 1; i <= TEST_ROUNDS; i++ ) {
|
||||
logAtIndent(
|
||||
1,
|
||||
// prettier-ignore
|
||||
`Suite: ${ formats.success( testSuite ) } (round ${ i } of ${ TEST_ROUNDS })`
|
||||
);
|
||||
|
||||
for ( const branch of branches ) {
|
||||
logAtIndent( 2, 'Branch:', formats.success( branch ) );
|
||||
|
||||
const sanitizedBranchName = sanitizeBranchName( branch );
|
||||
const runKey = `${ testSuite }_${ sanitizedBranchName }_round-${ i }`;
|
||||
const envDir = branchDirs[ branch ];
|
||||
|
||||
logAtIndent( 3, 'Starting environment' );
|
||||
await runShellScript( `${ wpEnvPath } start`, envDir );
|
||||
|
||||
logAtIndent( 3, 'Running tests' );
|
||||
await runTestSuite( testSuite, testRunnerDir, runKey );
|
||||
|
||||
logAtIndent( 3, 'Stopping environment' );
|
||||
await runShellScript( `${ wpEnvPath } stop`, envDir );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logAtIndent( 0, 'Calculating results' );
|
||||
|
||||
const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) =>
|
||||
file.endsWith( RESULTS_FILE_SUFFIX )
|
||||
);
|
||||
/** @type {Record<string,Record<string, Record<string, number>>>} */
|
||||
const results = {};
|
||||
|
||||
// Calculate medians from all rounds.
|
||||
for ( const testSuite of testSuites ) {
|
||||
logAtIndent( 1, 'Test suite:', formats.success( testSuite ) );
|
||||
|
||||
results[ testSuite ] = {};
|
||||
for ( const branch of branches ) {
|
||||
const sanitizedBranchName = sanitizeBranchName( branch );
|
||||
const resultsRounds = resultFiles
|
||||
.filter( ( file ) =>
|
||||
file.includes(
|
||||
`${ testSuite }_${ sanitizedBranchName }_round-`
|
||||
)
|
||||
)
|
||||
.map( ( file ) => {
|
||||
logAtIndent( 2, 'Reading from:', formats.success( file ) );
|
||||
return readJSONFile( file );
|
||||
} );
|
||||
|
||||
const metrics = Object.keys( resultsRounds[ 0 ] );
|
||||
results[ testSuite ][ branch ] = {};
|
||||
|
||||
for ( const metric of metrics ) {
|
||||
const values = resultsRounds
|
||||
.map( ( round ) => round[ metric ] )
|
||||
.filter( ( value ) => typeof value === 'number' );
|
||||
|
||||
const value = median( values );
|
||||
if ( value !== undefined ) {
|
||||
results[ testSuite ][ branch ][ metric ] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
const calculatedResultsPath = path.join(
|
||||
ARTIFACTS_PATH,
|
||||
testSuite + RESULTS_FILE_SUFFIX
|
||||
);
|
||||
|
||||
logAtIndent(
|
||||
2,
|
||||
'Saving curated results to:',
|
||||
formats.success( calculatedResultsPath )
|
||||
);
|
||||
fs.writeFileSync(
|
||||
calculatedResultsPath,
|
||||
JSON.stringify( results[ testSuite ], null, 2 )
|
||||
);
|
||||
}
|
||||
|
||||
logAtIndent( 0, 'Printing results' );
|
||||
|
||||
for ( const testSuite of testSuites ) {
|
||||
logAtIndent( 0, formats.success( testSuite ) );
|
||||
|
||||
// Invert the results so we can display them in a table.
|
||||
/** @type {Record<string, Record<string, string>>} */
|
||||
const invertedResult = {};
|
||||
for ( const [ branch, metrics ] of Object.entries(
|
||||
results[ testSuite ]
|
||||
) ) {
|
||||
for ( const [ metric, value ] of Object.entries( metrics ) ) {
|
||||
invertedResult[ metric ] = invertedResult[ metric ] || {};
|
||||
invertedResult[ metric ][ branch ] = `${ value } ms`;
|
||||
}
|
||||
}
|
||||
|
||||
// Print the results.
|
||||
console.table( invertedResult );
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
runPerformanceTests,
|
||||
};
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
/* eslint-disable no-console */
|
||||
const inquirer = require( 'inquirer' );
|
||||
const fs = require( 'fs' );
|
||||
const childProcess = require( 'child_process' );
|
||||
const path = require( 'path' );
|
||||
const chalk = require( 'chalk' );
|
||||
|
||||
/**
|
||||
* Utility to run a child script
|
||||
*
|
||||
* @typedef {NodeJS.ProcessEnv} Env
|
||||
*
|
||||
* @param {string} script Script to run.
|
||||
* @param {string=} cwd Working directory.
|
||||
* @param {Env=} env Additional environment variables to pass to the script.
|
||||
*/
|
||||
function runShellScript( script, cwd, env = {} ) {
|
||||
return new Promise( ( resolve, reject ) => {
|
||||
childProcess.exec(
|
||||
script,
|
||||
{
|
||||
cwd,
|
||||
env: {
|
||||
NO_CHECKS: 'true',
|
||||
PATH: process.env.PATH,
|
||||
HOME: process.env.HOME,
|
||||
USER: process.env.USER,
|
||||
...env,
|
||||
},
|
||||
},
|
||||
function ( error, stdout, stderr ) {
|
||||
if ( error ) {
|
||||
console.log( stdout ); // Sometimes the error message is thrown via stdout.
|
||||
console.log( stderr );
|
||||
reject( error );
|
||||
} else {
|
||||
resolve( true );
|
||||
}
|
||||
}
|
||||
);
|
||||
} );
|
||||
}
|
||||
|
||||
/**
|
||||
* Small utility used to read an uncached version of a JSON file
|
||||
*
|
||||
* @param {string} fileName
|
||||
*/
|
||||
function readJSONFile( fileName ) {
|
||||
const data = fs.readFileSync( fileName, 'utf8' );
|
||||
return JSON.parse( data );
|
||||
}
|
||||
|
||||
/**
|
||||
* Asks the user for a confirmation to continue or abort otherwise.
|
||||
*
|
||||
* @param {string} message Confirmation message.
|
||||
* @param {boolean} isDefault Default reply.
|
||||
* @param {string} abortMessage Abort message.
|
||||
*/
|
||||
async function askForConfirmation(
|
||||
message,
|
||||
isDefault = true,
|
||||
abortMessage = 'Aborting.'
|
||||
) {
|
||||
const { isReady } = await inquirer.prompt( [
|
||||
{
|
||||
type: 'confirm',
|
||||
name: 'isReady',
|
||||
default: isDefault,
|
||||
message,
|
||||
},
|
||||
] );
|
||||
|
||||
if ( ! isReady ) {
|
||||
chalk.log( chalk.bold.red( '\n' + abortMessage ) );
|
||||
process.exit( 1 );
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans the given directory and returns an array of file paths.
|
||||
*
|
||||
* @param {string} dir The path to the directory to scan.
|
||||
*
|
||||
* @return {string[]} An array of file paths.
|
||||
*/
|
||||
function getFilesFromDir( dir ) {
|
||||
if ( ! fs.existsSync( dir ) ) {
|
||||
console.log( 'Directory does not exist: ', dir );
|
||||
return [];
|
||||
}
|
||||
|
||||
return fs
|
||||
.readdirSync( dir, { withFileTypes: true } )
|
||||
.filter( ( dirent ) => dirent.isFile() )
|
||||
.map( ( dirent ) => path.join( dir, dirent.name ) );
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
askForConfirmation,
|
||||
readJSONFile,
|
||||
runShellScript,
|
||||
getFilesFromDir,
|
||||
};
|
||||
Loading…
Reference in New Issue