From 55f855a2e6d769b5ae44305b2772eb30d3e721df Mon Sep 17 00:00:00 2001 From: Vladimir Reznichenko Date: Fri, 13 Sep 2024 08:58:44 +0200 Subject: [PATCH] [dev] Monorepo: introduce reporting mode for perf utility. (#51366) Monorepo: introduce reporting mode for perf utility. --- pnpm-lock.yaml | 12 ++ tools/compare-perf/index.js | 11 +- tools/compare-perf/package.json | 8 +- tools/compare-perf/performance.js | 153 ++++---------------------- tools/compare-perf/process-reports.ts | 119 ++++++++++++++++++++ tools/compare-perf/tsconfig.json | 15 +++ tools/compare-perf/utils.js | 49 +++++++++ 7 files changed, 233 insertions(+), 134 deletions(-) create mode 100644 tools/compare-perf/process-reports.ts create mode 100644 tools/compare-perf/tsconfig.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3f31827f5ef..2a972c8f247 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4658,6 +4658,12 @@ importers: tools/compare-perf: dependencies: + '@tsconfig/node16': + specifier: ^1.0.4 + version: 1.0.4 + '@types/node': + specifier: ^16.18.68 + version: 16.18.68 '@wordpress/env': specifier: ^10.1.0 version: 10.5.0 @@ -4673,6 +4679,12 @@ importers: simple-git: specifier: ^3.21.0 version: 3.21.0 + ts-node: + specifier: ^10.9.2 + version: 10.9.2(@swc/core@1.3.100)(@types/node@16.18.68)(typescript@5.3.3) + tslib: + specifier: ^2.6.2 + version: 2.6.3 tools/monorepo-merge: dependencies: diff --git a/tools/compare-perf/index.js b/tools/compare-perf/index.js index d14680f9bf0..beca567b34c 100755 --- a/tools/compare-perf/index.js +++ b/tools/compare-perf/index.js @@ -15,12 +15,17 @@ const catchException = ( command ) => { }; }; -const ciOption = [ '-c, --ci', 'Run in CI (non interactive)' ]; - program .command( 'compare-performance [branches...]' ) .alias( 'perf' ) - .option( ...ciOption ) + .option( + '-c, --ci', + 'Run in CI (non interactive)' + ) + .option( + '--skip-benchmarking', + 'Skips benchmarking and gets straight to reporting phase (tests results already available)' + ) .option( '--rounds ', 'Run each test suite this many times for each branch; results are summarized, default = 1' diff --git a/tools/compare-perf/package.json b/tools/compare-perf/package.json index 430d64146e5..424decbe36d 100644 --- a/tools/compare-perf/package.json +++ b/tools/compare-perf/package.json @@ -7,15 +7,19 @@ "license": "GPLv2", "repository": "woocommerce/woocommerce", "scripts": { - "compare": "node index.js", + "compare": "node -r ts-node/register index.js", "log": "node log-to-codevitals.js" }, "dependencies": { + "@types/node": "^16.18.68", + "@tsconfig/node16": "^1.0.4", "@wordpress/env": "^10.1.0", "commander": "9.5.0", "chalk": "^4.1.2", "inquirer": "^7.1.0", - "simple-git": "^3.21.0" + "simple-git": "^3.21.0", + "ts-node": "^10.9.2", + "tslib": "^2.6.2" }, "engines": { "node": "^20.11.1", diff --git a/tools/compare-perf/performance.js b/tools/compare-perf/performance.js index 47315f382ef..527ad2a914e 100644 --- a/tools/compare-perf/performance.js +++ b/tools/compare-perf/performance.js @@ -13,71 +13,27 @@ const formats = { }; const { runShellScript, - readJSONFile, askForConfirmation, getFilesFromDir, + logAtIndent, + sanitizeBranchName, } = require( './utils' ); const config = require( './config' ); +const { processPerformanceReports } = require( './process-reports.ts' ); const ARTIFACTS_PATH = process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' ); -const RESULTS_FILE_SUFFIX = '.performance-results.json'; /** * @typedef WPPerformanceCommandOptions * - * @property {boolean=} ci Run on CI. - * @property {number=} rounds Run each test suite this many times for each branch. - * @property {string=} testsBranch The branch whose performance test files will be used for testing. - * @property {string=} wpVersion The WordPress version to be used as the base install for testing. + * @property {boolean=} ci Run on CI. + * @property {number=} rounds Run each test suite this many times for each branch. + * @property {string=} testsBranch The branch whose performance test files will be used for testing. + * @property {boolean=} skipBenchmarking Skip benchmarking and get to report processing (reports supplied from outside). + * @property {string=} wpVersion The WordPress version to be used as the base install for testing. */ -/** - * A logging helper for printing steps and their substeps. - * - * @param {number} indent Value to indent the log. - * @param {any} msg Message to log. - * @param {...any} args Rest of the arguments to pass to console.log. - */ -function logAtIndent( indent, msg, ...args ) { - const prefix = indent === 0 ? '▶ ' : '> '; - const newline = indent === 0 ? '\n' : ''; - return console.log( - newline + ' '.repeat( indent ) + prefix + msg, - ...args - ); -} - -/** - * Sanitizes branch name to be used in a path or a filename. - * - * @param {string} branch - * - * @return {string} Sanitized branch name. - */ -function sanitizeBranchName( branch ) { - return branch.replace( /[^a-zA-Z0-9-]/g, '-' ); -} - -/** - * Computes the median number from an array numbers. - * - * @param {number[]} array - * - * @return {number|undefined} Median value or undefined if array empty. - */ -function median( array ) { - if ( ! array || ! array.length ) return undefined; - - const numbers = [ ...array ].sort( ( a, b ) => a - b ); - const middleIndex = Math.floor( numbers.length / 2 ); - - if ( numbers.length % 2 === 0 ) { - return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2; - } - return numbers[ middleIndex ]; -} - /** * Runs the performance tests on the current branch. * @@ -106,6 +62,7 @@ async function runTestSuite( testSuite, testRunnerDir, runKey ) { */ async function runPerformanceTests( branches, options ) { const runningInCI = !! process.env.CI || !! options.ci; + const skipBenchmarking = !! options.skipBenchmarking; const TEST_ROUNDS = options.rounds || 1; // The default value doesn't work because commander provides an array. @@ -129,6 +86,20 @@ async function runPerformanceTests( branches, options ) { await askForConfirmation( 'Ready to go? ' ); } + if ( skipBenchmarking ) { + // When benchmarking is skipped, it's expected that artifacts folder contains reports for the branches. + // If so, we'll process reports and pick test suites as per current state of codebase. + const testSuites = getFilesFromDir( + path.resolve( __dirname, '../..' ) + config.testsPath + ).map( ( file ) => { + logAtIndent( 1, 'Found:', formats.success( file ) ); + return path.basename( file, '.spec.js' ); + } ); + + await processPerformanceReports( testSuites, branches ); + return; + } + logAtIndent( 0, 'Setting up' ); /** @@ -140,7 +111,6 @@ async function runPerformanceTests( branches, options ) { } const baseDir = path.join( os.tmpdir(), 'wp-performance-tests' ); - if ( fs.existsSync( baseDir ) ) { logAtIndent( 1, 'Removing existing files' ); fs.rmSync( baseDir, { recursive: true } ); @@ -350,82 +320,7 @@ async function runPerformanceTests( branches, options ) { } } - logAtIndent( 0, 'Calculating results' ); - - const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) => - file.endsWith( RESULTS_FILE_SUFFIX ) - ); - /** @type {Record>>} */ - const results = {}; - - // Calculate medians from all rounds. - for ( const testSuite of testSuites ) { - logAtIndent( 1, 'Test suite:', formats.success( testSuite ) ); - - results[ testSuite ] = {}; - for ( const branch of branches ) { - const sanitizedBranchName = sanitizeBranchName( branch ); - const resultsRounds = resultFiles - .filter( ( file ) => - file.includes( - `/${ testSuite }_${ sanitizedBranchName }_round-` - ) - ) - .map( ( file ) => { - logAtIndent( 2, 'Reading from:', formats.success( file ) ); - return readJSONFile( file ); - } ); - - const metrics = Object.keys( resultsRounds[ 0 ] ); - results[ testSuite ][ branch ] = {}; - - for ( const metric of metrics ) { - const values = resultsRounds - .map( ( round ) => round[ metric ] ) - .filter( ( value ) => typeof value === 'number' ); - - const value = median( values ); - if ( value !== undefined ) { - results[ testSuite ][ branch ][ metric ] = value; - } - } - } - const calculatedResultsPath = path.join( - ARTIFACTS_PATH, - testSuite + RESULTS_FILE_SUFFIX - ); - - logAtIndent( - 2, - 'Saving curated results to:', - formats.success( calculatedResultsPath ) - ); - fs.writeFileSync( - calculatedResultsPath, - JSON.stringify( results[ testSuite ], null, 2 ) - ); - } - - logAtIndent( 0, 'Printing results' ); - - for ( const testSuite of testSuites ) { - logAtIndent( 0, formats.success( testSuite ) ); - - // Invert the results so we can display them in a table. - /** @type {Record>} */ - const invertedResult = {}; - for ( const [ branch, metrics ] of Object.entries( - results[ testSuite ] - ) ) { - for ( const [ metric, value ] of Object.entries( metrics ) ) { - invertedResult[ metric ] = invertedResult[ metric ] || {}; - invertedResult[ metric ][ branch ] = `${ value } ms`; - } - } - - // Print the results. - console.table( invertedResult ); - } + await processPerformanceReports( testSuites, branches ); } module.exports = { diff --git a/tools/compare-perf/process-reports.ts b/tools/compare-perf/process-reports.ts new file mode 100644 index 00000000000..ec2684161e9 --- /dev/null +++ b/tools/compare-perf/process-reports.ts @@ -0,0 +1,119 @@ +/** + * External dependencies + */ +const bold = require( 'chalk' ); +const fs = require( 'fs' ); +const path = require( 'path' ); + +/** + * Internal dependencies + */ +const { + getFilesFromDir, + readJSONFile, + logAtIndent, + sanitizeBranchName, + median +} = require( './utils' ) ; + +const formats = { + success: bold.green, +}; + +const ARTIFACTS_PATH = + process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' ); +const RESULTS_FILE_SUFFIX = '.performance-results.json'; + +/** + * Calculates and prints results from the generated reports. + * + * @param {string[]} testSuites Test suites we are aiming. + * @param {string[]} branches Branches we are aiming. + */ +async function processPerformanceReports( + testSuites: string[], + branches: string[] +): Promise< void > { + logAtIndent( 0, 'Calculating results' ); + + const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( + ( file: string ) => file.endsWith( RESULTS_FILE_SUFFIX ) + ); + const results: Record< + string, + Record< string, Record< string, number > > + > = {}; + + // Calculate medians from all rounds. + for ( const testSuite of testSuites ) { + logAtIndent( 1, 'Test suite:', formats.success( testSuite ) ); + + results[ testSuite ] = {}; + for ( const branch of branches ) { + const sanitizedBranchName = sanitizeBranchName( branch ); + const resultsRounds: any[] = resultFiles + .filter( ( file: string ) => + file.includes( + `/${ testSuite }_${ sanitizedBranchName }_round-` + ) + ) + .map( ( file: string ) => { + logAtIndent( 2, 'Reading from:', formats.success( file ) ); + return readJSONFile( file ); + } ); + + const metrics = Object.keys( resultsRounds[ 0 ] ); + results[ testSuite ][ branch ] = {}; + + for ( const metric of metrics ) { + const values = resultsRounds + .map( ( round ) => round[ metric ] ) + .filter( ( value ) => typeof value === 'number' ); + + const value = median( values ); + if ( value !== undefined ) { + results[ testSuite ][ branch ][ metric ] = value; + } + } + } + const calculatedResultsPath = path.join( + ARTIFACTS_PATH, + testSuite + RESULTS_FILE_SUFFIX + ); + + logAtIndent( + 2, + 'Saving curated results to:', + formats.success( calculatedResultsPath ) + ); + fs.writeFileSync( + calculatedResultsPath, + JSON.stringify( results[ testSuite ], null, 2 ) + ); + } + + logAtIndent( 0, 'Printing results' ); + + for ( const testSuite of testSuites ) { + logAtIndent( 0, formats.success( testSuite ) ); + + // Invert the results, so we can display them in a table. + const invertedResult: Record< string, Record< string, string > > = {}; + for ( const [ branch, metrics ] of Object.entries( + results[ testSuite ] + ) ) { + for ( const [ metric, value ] of Object.entries( metrics ) ) { + invertedResult[ metric ] = invertedResult[ metric ] || {}; + invertedResult[ metric ][ branch ] = `${ value } ms`; + } + } + + // Print the results. + // eslint-disable-next-line no-console + console.table( invertedResult ); + } +} + +module.exports = { + processPerformanceReports, +}; diff --git a/tools/compare-perf/tsconfig.json b/tools/compare-perf/tsconfig.json new file mode 100644 index 00000000000..d491017bfb9 --- /dev/null +++ b/tools/compare-perf/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "./node_modules/@tsconfig/node16/tsconfig.json", + "compilerOptions": { + "module": "Node16", + "moduleResolution": "Node16", + "typeRoots": [ + "./typings", + "./node_modules/@types" + ] + }, + "ts-node": { + "transpileOnly": true, + "files": true, + }, +} diff --git a/tools/compare-perf/utils.js b/tools/compare-perf/utils.js index 2cca3b1a781..4a47cb41ded 100644 --- a/tools/compare-perf/utils.js +++ b/tools/compare-perf/utils.js @@ -97,9 +97,58 @@ function getFilesFromDir( dir ) { .map( ( dirent ) => path.join( dir, dirent.name ) ); } +/** + * A logging helper for printing steps and their substeps. + * + * @param {number} indent Value to indent the log. + * @param {any} msg Message to log. + * @param {...any} args Rest of the arguments to pass to console.log. + */ +function logAtIndent( indent, msg, ...args ) { + const prefix = indent === 0 ? '▶ ' : '> '; + const newline = indent === 0 ? '\n' : ''; + return console.log( + newline + ' '.repeat( indent ) + prefix + msg, + ...args + ); +} + +/** + * Sanitizes branch name to be used in a path or a filename. + * + * @param {string} branch + * + * @return {string} Sanitized branch name. + */ +function sanitizeBranchName( branch ) { + return branch.replace( /[^a-zA-Z0-9-]/g, '-' ); +} + +/** + * Computes the median number from an array numbers. + * + * @param {number[]} array + * + * @return {number|undefined} Median value or undefined if array empty. + */ +function median( array ) { + if ( ! array || ! array.length ) return undefined; + + const numbers = [ ...array ].sort( ( a, b ) => a - b ); + const middleIndex = Math.floor( numbers.length / 2 ); + + if ( numbers.length % 2 === 0 ) { + return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2; + } + return numbers[ middleIndex ]; +} + module.exports = { askForConfirmation, readJSONFile, runShellScript, getFilesFromDir, + logAtIndent, + sanitizeBranchName, + median };