Performance: Add a tool and a CI workflow to compare performance between PR and trunk and track metrics on trunk

.github/workflows/metrics.yml

@@ -0,0 +1,53 @@
+name: Metrics Tracking
+
+on:
+    pull_request:
+    push:
+        branches: [trunk]
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+    cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+    metrics:
+        name: Run metrics tests
+        runs-on: ubuntu-20.04
+        permissions:
+            contents: read
+        env:
+            WP_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts
+
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Setup WooCommerce Monorepo
+              uses: ./.github/actions/setup-woocommerce-monorepo
+              with:
+                install: '@woocommerce/plugin-woocommerce...'
+                build: '@woocommerce/plugin-woocommerce'
+
+            - name: Compare performance with trunk
+              if: github.event_name == 'pull_request'
+              run: cd tools/compare-perf && pnpm run perf $GITHUB_SHA trunk --tests-branch $GITHUB_SHA
+
+            - name: Compare performance with base branch
+              if: github.event_name == 'push'
+              # The base hash used here need to be a commit that is compatible with the current WP version
+              # The current one is 2f6ca66e00b3666b2567877ae67cbfb5b6ce171a 
+              # it needs to be updated every time it becomes unsupported by the current wp-env (WP version).
+              # It is used as a base comparison point to avoid fluctuation in the performance metrics.
+              run: |
+                  WP_VERSION=$(awk -F ': ' '/^Tested up to/{print $2}' readme.txt)
+                  IFS=. read -ra WP_VERSION_ARRAY <<< "$WP_VERSION"
+                  WP_MAJOR="${WP_VERSION_ARRAY[0]}.${WP_VERSION_ARRAY[1]}"
+                  cd tools/compare-perf && pnpm run perf $GITHUB_SHA 2f6ca66e00b3666b2567877ae67cbfb5b6ce171a --tests-branch $GITHUB_SHA --wp-version "$WP_MAJOR"
+
+            - name: Archive performance results
+              if: success()
+              uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+              with:
+                  name: performance-results
+                  path: ${{ env.WP_ARTIFACTS_PATH }}/*.performance-results*.json

.gitignore

@@ -108,3 +108,4 @@ changes.json
 # Metrics tests
 /artifacts
 /plugins/*/artifacts
+/tools/*/artifacts

pnpm-lock.yaml

@@ -4610,6 +4610,24 @@ importers:
         specifier: 0.14.1
         version: 0.14.1
 
+  tools/compare-perf:
+    dependencies:
+      '@wordpress/env':
+        specifier: ^8.13.0
+        version: 8.13.0
+      chalk:
+        specifier: ^4.1.2
+        version: 4.1.2
+      commander:
+        specifier: 9.5.0
+        version: 9.5.0
+      inquirer:
+        specifier: ^7.1.0
+        version: 7.3.3
+      simple-git:
+        specifier: 3.5.0
+        version: 3.5.0
+
   tools/create-extension:
     dependencies:
       chalk:
@@ -14670,7 +14688,6 @@ packages:
   /@sindresorhus/is@4.6.0:
     resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==}
     engines: {node: '>=10'}
-    dev: true
 
   /@sindresorhus/is@5.6.0:
     resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==}
@@ -18448,7 +18465,6 @@ packages:
     engines: {node: '>=10'}
     dependencies:
       defer-to-connect: 2.0.1
-    dev: true
 
   /@szmarczak/http-timer@5.0.1:
     resolution: {integrity: sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==}
@@ -18744,7 +18760,6 @@ packages:
       '@types/keyv': 3.1.4
       '@types/node': 20.10.4
       '@types/responselike': 1.0.3
-    dev: true
 
   /@types/cheerio@0.22.35:
     resolution: {integrity: sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==}
@@ -23250,7 +23265,6 @@ packages:
       yargs: 17.7.2
     transitivePeerDependencies:
       - supports-color
-    dev: true
 
   /@wordpress/env@8.2.0:
     resolution: {integrity: sha512-MGf2TJD6MbBjDn/+feGAcXnh6ct9Y/aCjIZTtbMI+b+pXAPh0RzR7Q5D6KMgKUNcGeL6nNQmEUUSSPuzV2UWfQ==}
@@ -27864,7 +27878,7 @@ packages:
     resolution: {integrity: sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==}
     dependencies:
       base64-js: 1.5.1
-      ieee754: 1.1.13
+      ieee754: 1.2.1
       isarray: 1.0.0
     dev: true
 
@@ -28047,7 +28061,6 @@ packages:
   /cacheable-lookup@5.0.4:
     resolution: {integrity: sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==}
     engines: {node: '>=10.6.0'}
-    dev: true
 
   /cacheable-lookup@7.0.0:
     resolution: {integrity: sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==}
@@ -28078,7 +28091,6 @@ packages:
       lowercase-keys: 2.0.0
       normalize-url: 6.1.0
       responselike: 2.0.1
-    dev: true
 
   /call-bind@1.0.5:
     resolution: {integrity: sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==}
@@ -28772,7 +28784,6 @@ packages:
     resolution: {integrity: sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==}
     dependencies:
       mimic-response: 1.0.1
-    dev: true
 
   /clone-stats@1.0.0:
     resolution: {integrity: sha512-au6ydSpg6nsrigcZ4m8Bc9hxjeW+GJ8xh5G3BJCMt4WXe1H10UNaVOamqQTmrx1kjVuxAHIQSNU6hY4Nsn9/ag==}
@@ -29269,7 +29280,6 @@ packages:
 
   /copy-dir@1.3.0:
     resolution: {integrity: sha512-Q4+qBFnN4bwGwvtXXzbp4P/4iNk0MaiGAzvQ8OiMtlLjkIKjmNN689uVzShSM0908q7GoFHXIPx4zi75ocoaHw==}
-    dev: true
 
   /copy-to-clipboard@3.3.3:
     resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==}
@@ -30789,7 +30799,6 @@ packages:
   /docker-compose@0.22.2:
     resolution: {integrity: sha512-iXWb5+LiYmylIMFXvGTYsjI1F+Xyx78Jm/uj1dxwwZLbWkUdH6yOXY5Nr3RjbYX15EgbGJCq78d29CmWQQQMPg==}
     engines: {node: '>= 6.0.0'}
-    dev: true
 
   /doctrine@2.1.0:
     resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==}
@@ -34822,7 +34831,6 @@ packages:
       lowercase-keys: 2.0.0
       p-cancelable: 2.1.1
       responselike: 2.0.1
-    dev: true
 
   /got@12.6.1:
     resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==}
@@ -35780,7 +35788,6 @@ packages:
     dependencies:
       quick-lru: 5.1.1
       resolve-alpn: 1.2.1
-    dev: true
 
   /http2-wrapper@2.2.1:
     resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==}
@@ -40636,7 +40643,6 @@ packages:
     engines: {node: '>=8'}
     dependencies:
       chalk: 2.4.2
-    dev: true
 
   /log-symbols@4.1.0:
     resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==}
@@ -40713,7 +40719,6 @@ packages:
   /lowercase-keys@2.0.0:
     resolution: {integrity: sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==}
     engines: {node: '>=8'}
-    dev: true
 
   /lowercase-keys@3.0.0:
     resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==}
@@ -41699,7 +41704,6 @@ packages:
   /mimic-response@1.0.1:
     resolution: {integrity: sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==}
     engines: {node: '>=4'}
-    dev: true
 
   /mimic-response@3.1.0:
     resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
@@ -42508,7 +42512,6 @@ packages:
   /normalize-url@6.1.0:
     resolution: {integrity: sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==}
     engines: {node: '>=10'}
-    dev: true
 
   /normalize-url@8.0.0:
     resolution: {integrity: sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==}
@@ -43093,7 +43096,6 @@ packages:
       mute-stream: 0.0.8
       strip-ansi: 6.0.1
       wcwidth: 1.0.1
-    dev: true
 
   /ora@5.4.1:
     resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==}
@@ -43145,7 +43147,6 @@ packages:
   /p-cancelable@2.1.1:
     resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==}
     engines: {node: '>=8'}
-    dev: true
 
   /p-cancelable@3.0.0:
     resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==}
@@ -47616,7 +47617,6 @@ packages:
     resolution: {integrity: sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw==}
     dependencies:
       lowercase-keys: 2.0.0
-    dev: true
 
   /responselike@3.0.0:
     resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==}
@@ -48411,6 +48411,16 @@ packages:
     transitivePeerDependencies:
       - supports-color
 
+  /simple-git@3.5.0:
+    resolution: {integrity: sha512-fZsaq5nzdxQRhMNs6ESGLpMUHoL5GRP+boWPhq9pMYMKwOGZV2jHOxi8AbFFA2Y/6u4kR99HoULizSbpzaODkA==}
+    dependencies:
+      '@kwsites/file-exists': 1.1.1
+      '@kwsites/promise-deferred': 1.1.1
+      debug: 4.3.4(supports-color@9.4.0)
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /simple-html-tokenizer@0.5.11:
     resolution: {integrity: sha512-C2WEK/Z3HoSFbYq8tI7ni3eOo/NneSPRoPpcM7WdLjFOArFuyXEjAoCdOC3DgMfRyziZQ1hCNR4mrNdWEvD0og==}
 

pnpm-workspace.yaml

@@ -4,6 +4,7 @@ packages:
     - 'plugins/woocommerce/client/legacy'
     - 'tools/monorepo-merge'
     - 'tools/code-analyzer'
+    - 'tools/compare-perf'
     - 'tools/create-extension'
     - 'tools/package-release'
     - 'tools/cherry-pick'

tools/compare-perf/config.js

@@ -0,0 +1,11 @@
+const config = {
+	gitRepositoryURL: 'https://github.com/woocommerce/woocommerce.git',
+	setupCommand:
+		'npm install -g pnpm && pnpm install &> /dev/null && pnpm build  &> /dev/null && cd plugins/woocommerce && pnpm exec playwright install chromium',
+	pluginPath: '/plugins/woocommerce',
+	testsPath: '/plugins/woocommerce/tests/metrics/specs',
+	testCommand:
+		'npm install -g pnpm && cd plugins/woocommerce && pnpm test:metrics',
+};
+
+module.exports = config;

tools/compare-perf/index.js

@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+/* eslint-disable no-console */
+
+const program = require( 'commander' );
+const { runPerformanceTests } = require( './performance' );
+
+const catchException = ( command ) => {
+	return async ( ...args ) => {
+		try {
+			await command( ...args );
+		} catch ( error ) {
+			console.error( error );
+			process.exitCode = 1;
+		}
+	};
+};
+
+const ciOption = [ '-c, --ci', 'Run in CI (non interactive)' ];
+
+program
+	.command( 'compare-performance [branches...]' )
+	.alias( 'perf' )
+	.option( ...ciOption )
+	.option(
+		'--rounds <count>',
+		'Run each test suite this many times for each branch; results are summarized, default = 1'
+	)
+	.option(
+		'--tests-branch <branch>',
+		"Use this branch's performance test files"
+	)
+	.option(
+		'--wp-version <version>',
+		'Specify a WordPress version on which to test all branches'
+	)
+	.description(
+		'Runs performance tests on two separate branches and outputs the result'
+	)
+	.action( catchException( runPerformanceTests ) );
+
+program.parse( process.argv );

tools/compare-perf/package.json

@@ -0,0 +1,23 @@
+{
+	"name": "compare-perf",
+	"version": "0.0.1",
+	"description": "A tool to compare performance accross tow branches in WooCommerce Monorepo.",
+	"author": "Automattic",
+	"homepage": "https://github.com/woocommerce/woocommerce",
+	"license": "GPLv2",
+	"repository": "woocommerce/woocommerce",
+	"scripts": {
+		"test": "node index.js"
+	},
+	"dependencies": {
+		"@wordpress/env": "^8.13.0",
+		"commander": "9.5.0",
+		"chalk": "^4.1.2",
+		"inquirer": "^7.1.0",
+		"simple-git": "3.5.0"
+	},
+	"engines": {
+		"node": "^16.14.1",
+		"pnpm": "^8.6.7"
+	}
+}

tools/compare-perf/performance.js

@@ -0,0 +1,429 @@
+/* eslint-disable no-console */
+const os = require( 'os' );
+const fs = require( 'fs' );
+const path = require( 'path' );
+const SimpleGit = require( 'simple-git' );
+const chalk = require( 'chalk' );
+
+const formats = {
+	title: chalk.bold,
+	error: chalk.bold.red,
+	warning: chalk.bold.keyword( 'orange' ),
+	success: chalk.bold.green,
+};
+const {
+	runShellScript,
+	readJSONFile,
+	askForConfirmation,
+	getFilesFromDir,
+} = require( './utils' );
+const config = require( './config' );
+
+const ARTIFACTS_PATH =
+	process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
+const RESULTS_FILE_SUFFIX = '.performance-results.json';
+
+/**
+ * @typedef WPPerformanceCommandOptions
+ *
+ * @property {boolean=} ci          Run on CI.
+ * @property {number=}  rounds      Run each test suite this many times for each branch.
+ * @property {string=}  testsBranch The branch whose performance test files will be used for testing.
+ * @property {string=}  wpVersion   The WordPress version to be used as the base install for testing.
+ */
+
+/**
+ * A logging helper for printing steps and their substeps.
+ *
+ * @param {number} indent Value to indent the log.
+ * @param {any}    msg    Message to log.
+ * @param {...any} args   Rest of the arguments to pass to console.log.
+ */
+function logAtIndent( indent, msg, ...args ) {
+	const prefix = indent === 0 ? '▶ ' : '> ';
+	const newline = indent === 0 ? '\n' : '';
+	return console.log(
+		newline + '    '.repeat( indent ) + prefix + msg,
+		...args
+	);
+}
+
+/**
+ * Sanitizes branch name to be used in a path or a filename.
+ *
+ * @param {string} branch
+ *
+ * @return {string} Sanitized branch name.
+ */
+function sanitizeBranchName( branch ) {
+	return branch.replace( /[^a-zA-Z0-9-]/g, '-' );
+}
+
+/**
+ * Computes the median number from an array numbers.
+ *
+ * @param {number[]} array
+ *
+ * @return {number|undefined} Median value or undefined if array empty.
+ */
+function median( array ) {
+	if ( ! array || ! array.length ) return undefined;
+
+	const numbers = [ ...array ].sort( ( a, b ) => a - b );
+	const middleIndex = Math.floor( numbers.length / 2 );
+
+	if ( numbers.length % 2 === 0 ) {
+		return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
+	}
+	return numbers[ middleIndex ];
+}
+
+/**
+ * Runs the performance tests on the current branch.
+ *
+ * @param {string} testSuite     Name of the tests set.
+ * @param {string} testRunnerDir Path to the performance tests' clone.
+ * @param {string} runKey        Unique identifier for the test run.
+ */
+async function runTestSuite( testSuite, testRunnerDir, runKey ) {
+	await runShellScript(
+		`${ config.testCommand } ${ testSuite }`,
+		testRunnerDir,
+		{
+			...process.env,
+			PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1',
+			WP_ARTIFACTS_PATH: ARTIFACTS_PATH,
+			RESULTS_ID: runKey,
+		}
+	);
+}
+
+/**
+ * Runs the performances tests on an array of branches and output the result.
+ *
+ * @param {string[]}                    branches Branches to compare
+ * @param {WPPerformanceCommandOptions} options  Command options.
+ */
+async function runPerformanceTests( branches, options ) {
+	const runningInCI = !! process.env.CI || !! options.ci;
+	const TEST_ROUNDS = options.rounds || 1;
+
+	// The default value doesn't work because commander provides an array.
+	if ( branches.length === 0 ) {
+		branches = [ 'trunk' ];
+	}
+
+	console.log( formats.title( '\n💃 Performance Tests 🕺' ) );
+	console.log(
+		'\nWelcome! This tool runs the performance tests on multiple branches and displays a comparison table.'
+	);
+
+	if ( ! runningInCI ) {
+		console.log(
+			formats.warning(
+				'\nIn order to run the tests, the tool is going to load a WordPress testing environment.' +
+					'\nMake sure to disable your own environment and testing ports before continuing.\n'
+			)
+		);
+
+		await askForConfirmation( 'Ready to go? ' );
+	}
+
+	logAtIndent( 0, 'Setting up' );
+
+	/**
+	 * @type {string[]} git refs against which to run tests;
+	 *                  could be commit SHA, branch name, tag, etc...
+	 */
+	if ( branches.length < 2 ) {
+		throw new Error( `Need at least two git refs to run` );
+	}
+
+	const baseDir = path.join( os.tmpdir(), 'wp-performance-tests' );
+
+	if ( fs.existsSync( baseDir ) ) {
+		logAtIndent( 1, 'Removing existing files' );
+		fs.rmSync( baseDir, { recursive: true } );
+	}
+
+	logAtIndent( 1, 'Creating base directory:', formats.success( baseDir ) );
+	fs.mkdirSync( baseDir );
+
+	logAtIndent( 1, 'Setting up repository' );
+	const sourceDir = path.join( baseDir, 'source' );
+
+	logAtIndent( 2, 'Creating directory:', formats.success( sourceDir ) );
+	fs.mkdirSync( sourceDir );
+
+	const sourceGit = SimpleGit( sourceDir );
+	logAtIndent(
+		2,
+		'Initializing:',
+		formats.success( config.gitRepositoryURL )
+	);
+	await sourceGit
+		.raw( 'init' )
+		.raw( 'remote', 'add', 'origin', config.gitRepositoryURL );
+
+	for ( const [ i, branch ] of branches.entries() ) {
+		logAtIndent(
+			2,
+			`Fetching environment branch (${ i + 1 } of ${ branches.length }):`,
+			formats.success( branch )
+		);
+		await sourceGit.raw( 'fetch', '--depth=1', 'origin', branch );
+	}
+
+	const testRunnerBranch = options.testsBranch || branches[ 0 ];
+	if ( options.testsBranch && ! branches.includes( options.testsBranch ) ) {
+		logAtIndent(
+			2,
+			'Fetching test runner branch:',
+			formats.success( options.testsBranch )
+		);
+		await sourceGit.raw(
+			'fetch',
+			'--depth=1',
+			'origin',
+			options.testsBranch
+		);
+	} else {
+		logAtIndent(
+			2,
+			'Using test runner branch:',
+			formats.success( testRunnerBranch )
+		);
+	}
+
+	logAtIndent( 1, 'Setting up test runner' );
+
+	const testRunnerDir = path.join( baseDir + '/tests' );
+
+	logAtIndent( 2, 'Copying source to:', formats.success( testRunnerDir ) );
+	await runShellScript( `cp -R  ${ sourceDir } ${ testRunnerDir }` );
+
+	logAtIndent(
+		2,
+		'Checking out branch:',
+		formats.success( testRunnerBranch )
+	);
+	await SimpleGit( testRunnerDir ).raw( 'checkout', testRunnerBranch );
+
+	logAtIndent( 2, 'Installing dependencies and building' );
+	await runShellScript(
+		`bash -c "source $HOME/.nvm/nvm.sh && nvm install && ${ config.setupCommand }"`,
+		testRunnerDir
+	);
+
+	logAtIndent( 1, 'Setting up test environments' );
+
+	const envsDir = path.join( baseDir, 'environments' );
+	logAtIndent( 2, 'Creating parent directory:', formats.success( envsDir ) );
+	fs.mkdirSync( envsDir );
+
+	let wpZipUrl = null;
+	if ( options.wpVersion ) {
+		// In order to match the topology of ZIP files at wp.org, remap .0
+		// patch versions to major versions:
+		//
+		//     5.7   -> 5.7   (unchanged)
+		//     5.7.0 -> 5.7   (changed)
+		//     5.7.2 -> 5.7.2 (unchanged)
+		const zipVersion = options.wpVersion.replace( /^(\d+\.\d+).0/, '$1' );
+		wpZipUrl = `https://wordpress.org/wordpress-${ zipVersion }.zip`;
+	}
+
+	const branchDirs = {};
+	for ( const branch of branches ) {
+		logAtIndent( 2, 'Branch:', formats.success( branch ) );
+		const sanitizedBranchName = sanitizeBranchName( branch );
+		const envDir = path.join( envsDir, sanitizedBranchName );
+
+		logAtIndent( 3, 'Creating directory:', formats.success( envDir ) );
+		fs.mkdirSync( envDir );
+		branchDirs[ branch ] = envDir;
+		const buildDir = path.join( envDir, 'plugin' );
+
+		logAtIndent( 3, 'Copying source to:', formats.success( buildDir ) );
+		await runShellScript( `cp -R ${ sourceDir } ${ buildDir }` );
+
+		logAtIndent( 3, 'Checking out:', formats.success( branch ) );
+		await SimpleGit( buildDir ).raw( 'checkout', branch );
+
+		logAtIndent( 3, 'Installing dependencies and building' );
+		await runShellScript(
+			`bash -c "source $HOME/.nvm/nvm.sh && nvm install && ${ config.setupCommand }"`,
+			buildDir
+		);
+
+		const wpEnvConfigPath = path.join( envDir, '.wp-env.json' );
+
+		logAtIndent(
+			3,
+			'Saving wp-env config to:',
+			formats.success( wpEnvConfigPath )
+		);
+
+		fs.writeFileSync(
+			wpEnvConfigPath,
+			JSON.stringify(
+				{
+					config: {
+						WP_DEBUG: false,
+						SCRIPT_DEBUG: false,
+					},
+					core: wpZipUrl || 'WordPress/WordPress',
+					plugins: [ buildDir + config.pluginPath ],
+					themes: [
+						// Ideally this should be a fixed version of the theme.
+						// And it should be enabled in the tests suite.
+						'https://downloads.wordpress.org/theme/twentynineteen.zip',
+					],
+					env: {
+						tests: {
+							port: 8086,
+						},
+					},
+				},
+				null,
+				2
+			),
+			'utf8'
+		);
+	}
+
+	logAtIndent( 0, 'Looking for test files' );
+
+	const testSuites = getFilesFromDir(
+		path.join( testRunnerDir, config.testsPath )
+	).map( ( file ) => {
+		logAtIndent( 1, 'Found:', formats.success( file ) );
+		return path.basename( file, '.spec.js' );
+	} );
+
+	logAtIndent( 0, 'Running tests' );
+
+	if ( wpZipUrl ) {
+		logAtIndent(
+			1,
+			'Using:',
+			formats.success( `WordPress v${ options.wpVersion }` )
+		);
+	} else {
+		logAtIndent( 1, 'Using:', formats.success( 'WordPress trunk' ) );
+	}
+
+	// TODO: change this to tools/compare-perf/node_modules/.bin/wp-env
+	const wpEnvPath = path.join(
+		testRunnerDir,
+		'plugins/woocommerce/node_modules/.bin/wp-env'
+	);
+
+	for ( const testSuite of testSuites ) {
+		for ( let i = 1; i <= TEST_ROUNDS; i++ ) {
+			logAtIndent(
+				1,
+				// prettier-ignore
+				`Suite: ${ formats.success( testSuite ) } (round ${ i } of ${ TEST_ROUNDS })`
+			);
+
+			for ( const branch of branches ) {
+				logAtIndent( 2, 'Branch:', formats.success( branch ) );
+
+				const sanitizedBranchName = sanitizeBranchName( branch );
+				const runKey = `${ testSuite }_${ sanitizedBranchName }_round-${ i }`;
+				const envDir = branchDirs[ branch ];
+
+				logAtIndent( 3, 'Starting environment' );
+				await runShellScript( `${ wpEnvPath } start`, envDir );
+
+				logAtIndent( 3, 'Running tests' );
+				await runTestSuite( testSuite, testRunnerDir, runKey );
+
+				logAtIndent( 3, 'Stopping environment' );
+				await runShellScript( `${ wpEnvPath } stop`, envDir );
+			}
+		}
+	}
+
+	logAtIndent( 0, 'Calculating results' );
+
+	const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) =>
+		file.endsWith( RESULTS_FILE_SUFFIX )
+	);
+	/** @type {Record<string,Record<string, Record<string, number>>>} */
+	const results = {};
+
+	// Calculate medians from all rounds.
+	for ( const testSuite of testSuites ) {
+		logAtIndent( 1, 'Test suite:', formats.success( testSuite ) );
+
+		results[ testSuite ] = {};
+		for ( const branch of branches ) {
+			const sanitizedBranchName = sanitizeBranchName( branch );
+			const resultsRounds = resultFiles
+				.filter( ( file ) =>
+					file.includes(
+						`${ testSuite }_${ sanitizedBranchName }_round-`
+					)
+				)
+				.map( ( file ) => {
+					logAtIndent( 2, 'Reading from:', formats.success( file ) );
+					return readJSONFile( file );
+				} );
+
+			const metrics = Object.keys( resultsRounds[ 0 ] );
+			results[ testSuite ][ branch ] = {};
+
+			for ( const metric of metrics ) {
+				const values = resultsRounds
+					.map( ( round ) => round[ metric ] )
+					.filter( ( value ) => typeof value === 'number' );
+
+				const value = median( values );
+				if ( value !== undefined ) {
+					results[ testSuite ][ branch ][ metric ] = value;
+				}
+			}
+		}
+		const calculatedResultsPath = path.join(
+			ARTIFACTS_PATH,
+			testSuite + RESULTS_FILE_SUFFIX
+		);
+
+		logAtIndent(
+			2,
+			'Saving curated results to:',
+			formats.success( calculatedResultsPath )
+		);
+		fs.writeFileSync(
+			calculatedResultsPath,
+			JSON.stringify( results[ testSuite ], null, 2 )
+		);
+	}
+
+	logAtIndent( 0, 'Printing results' );
+
+	for ( const testSuite of testSuites ) {
+		logAtIndent( 0, formats.success( testSuite ) );
+
+		// Invert the results so we can display them in a table.
+		/** @type {Record<string, Record<string, string>>} */
+		const invertedResult = {};
+		for ( const [ branch, metrics ] of Object.entries(
+			results[ testSuite ]
+		) ) {
+			for ( const [ metric, value ] of Object.entries( metrics ) ) {
+				invertedResult[ metric ] = invertedResult[ metric ] || {};
+				invertedResult[ metric ][ branch ] = `${ value } ms`;
+			}
+		}
+
+		// Print the results.
+		console.table( invertedResult );
+	}
+}
+
+module.exports = {
+	runPerformanceTests,
+};

tools/compare-perf/utils.js

@@ -0,0 +1,105 @@
+/* eslint-disable no-console */
+const inquirer = require( 'inquirer' );
+const fs = require( 'fs' );
+const childProcess = require( 'child_process' );
+const path = require( 'path' );
+const chalk = require( 'chalk' );
+
+/**
+ * Utility to run a child script
+ *
+ * @typedef {NodeJS.ProcessEnv} Env
+ *
+ * @param {string}  script Script to run.
+ * @param {string=} cwd    Working directory.
+ * @param {Env=}    env    Additional environment variables to pass to the script.
+ */
+function runShellScript( script, cwd, env = {} ) {
+	return new Promise( ( resolve, reject ) => {
+		childProcess.exec(
+			script,
+			{
+				cwd,
+				env: {
+					NO_CHECKS: 'true',
+					PATH: process.env.PATH,
+					HOME: process.env.HOME,
+					USER: process.env.USER,
+					...env,
+				},
+			},
+			function ( error, stdout, stderr ) {
+				if ( error ) {
+					console.log( stdout ); // Sometimes the error message is thrown via stdout.
+					console.log( stderr );
+					reject( error );
+				} else {
+					resolve( true );
+				}
+			}
+		);
+	} );
+}
+
+/**
+ * Small utility used to read an uncached version of a JSON file
+ *
+ * @param {string} fileName
+ */
+function readJSONFile( fileName ) {
+	const data = fs.readFileSync( fileName, 'utf8' );
+	return JSON.parse( data );
+}
+
+/**
+ * Asks the user for a confirmation to continue or abort otherwise.
+ *
+ * @param {string}  message      Confirmation message.
+ * @param {boolean} isDefault    Default reply.
+ * @param {string}  abortMessage Abort message.
+ */
+async function askForConfirmation(
+	message,
+	isDefault = true,
+	abortMessage = 'Aborting.'
+) {
+	const { isReady } = await inquirer.prompt( [
+		{
+			type: 'confirm',
+			name: 'isReady',
+			default: isDefault,
+			message,
+		},
+	] );
+
+	if ( ! isReady ) {
+		chalk.log( chalk.bold.red( '\n' + abortMessage ) );
+		process.exit( 1 );
+	}
+}
+
+/**
+ * Scans the given directory and returns an array of file paths.
+ *
+ * @param {string} dir The path to the directory to scan.
+ *
+ * @return {string[]} An array of file paths.
+ */
+function getFilesFromDir( dir ) {
+	if ( ! fs.existsSync( dir ) ) {
+		console.log( 'Directory does not exist: ', dir );
+		return [];
+	}
+
+	return fs
+		.readdirSync( dir, { withFileTypes: true } )
+		.filter( ( dirent ) => dirent.isFile() )
+		.map( ( dirent ) => path.join( dir, dirent.name ) );
+}
+
+module.exports = {
+	askForConfirmation,
+	readJSONFile,
+	runShellScript,
+	getFilesFromDir,
+};