[dev] Monorepo: introduce reporting mode for perf utility. (#51366)
Monorepo: introduce reporting mode for perf utility.
This commit is contained in:
parent
4dad4a9265
commit
55f855a2e6
|
@ -4658,6 +4658,12 @@ importers:
|
||||||
|
|
||||||
tools/compare-perf:
|
tools/compare-perf:
|
||||||
dependencies:
|
dependencies:
|
||||||
|
'@tsconfig/node16':
|
||||||
|
specifier: ^1.0.4
|
||||||
|
version: 1.0.4
|
||||||
|
'@types/node':
|
||||||
|
specifier: ^16.18.68
|
||||||
|
version: 16.18.68
|
||||||
'@wordpress/env':
|
'@wordpress/env':
|
||||||
specifier: ^10.1.0
|
specifier: ^10.1.0
|
||||||
version: 10.5.0
|
version: 10.5.0
|
||||||
|
@ -4673,6 +4679,12 @@ importers:
|
||||||
simple-git:
|
simple-git:
|
||||||
specifier: ^3.21.0
|
specifier: ^3.21.0
|
||||||
version: 3.21.0
|
version: 3.21.0
|
||||||
|
ts-node:
|
||||||
|
specifier: ^10.9.2
|
||||||
|
version: 10.9.2(@swc/core@1.3.100)(@types/node@16.18.68)(typescript@5.3.3)
|
||||||
|
tslib:
|
||||||
|
specifier: ^2.6.2
|
||||||
|
version: 2.6.3
|
||||||
|
|
||||||
tools/monorepo-merge:
|
tools/monorepo-merge:
|
||||||
dependencies:
|
dependencies:
|
||||||
|
|
|
@ -15,12 +15,17 @@ const catchException = ( command ) => {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
const ciOption = [ '-c, --ci', 'Run in CI (non interactive)' ];
|
|
||||||
|
|
||||||
program
|
program
|
||||||
.command( 'compare-performance [branches...]' )
|
.command( 'compare-performance [branches...]' )
|
||||||
.alias( 'perf' )
|
.alias( 'perf' )
|
||||||
.option( ...ciOption )
|
.option(
|
||||||
|
'-c, --ci',
|
||||||
|
'Run in CI (non interactive)'
|
||||||
|
)
|
||||||
|
.option(
|
||||||
|
'--skip-benchmarking',
|
||||||
|
'Skips benchmarking and gets straight to reporting phase (tests results already available)'
|
||||||
|
)
|
||||||
.option(
|
.option(
|
||||||
'--rounds <count>',
|
'--rounds <count>',
|
||||||
'Run each test suite this many times for each branch; results are summarized, default = 1'
|
'Run each test suite this many times for each branch; results are summarized, default = 1'
|
||||||
|
|
|
@ -7,15 +7,19 @@
|
||||||
"license": "GPLv2",
|
"license": "GPLv2",
|
||||||
"repository": "woocommerce/woocommerce",
|
"repository": "woocommerce/woocommerce",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"compare": "node index.js",
|
"compare": "node -r ts-node/register index.js",
|
||||||
"log": "node log-to-codevitals.js"
|
"log": "node log-to-codevitals.js"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@types/node": "^16.18.68",
|
||||||
|
"@tsconfig/node16": "^1.0.4",
|
||||||
"@wordpress/env": "^10.1.0",
|
"@wordpress/env": "^10.1.0",
|
||||||
"commander": "9.5.0",
|
"commander": "9.5.0",
|
||||||
"chalk": "^4.1.2",
|
"chalk": "^4.1.2",
|
||||||
"inquirer": "^7.1.0",
|
"inquirer": "^7.1.0",
|
||||||
"simple-git": "^3.21.0"
|
"simple-git": "^3.21.0",
|
||||||
|
"ts-node": "^10.9.2",
|
||||||
|
"tslib": "^2.6.2"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": "^20.11.1",
|
"node": "^20.11.1",
|
||||||
|
|
|
@ -13,15 +13,16 @@ const formats = {
|
||||||
};
|
};
|
||||||
const {
|
const {
|
||||||
runShellScript,
|
runShellScript,
|
||||||
readJSONFile,
|
|
||||||
askForConfirmation,
|
askForConfirmation,
|
||||||
getFilesFromDir,
|
getFilesFromDir,
|
||||||
|
logAtIndent,
|
||||||
|
sanitizeBranchName,
|
||||||
} = require( './utils' );
|
} = require( './utils' );
|
||||||
const config = require( './config' );
|
const config = require( './config' );
|
||||||
|
const { processPerformanceReports } = require( './process-reports.ts' );
|
||||||
|
|
||||||
const ARTIFACTS_PATH =
|
const ARTIFACTS_PATH =
|
||||||
process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
|
process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
|
||||||
const RESULTS_FILE_SUFFIX = '.performance-results.json';
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @typedef WPPerformanceCommandOptions
|
* @typedef WPPerformanceCommandOptions
|
||||||
|
@ -29,55 +30,10 @@ const RESULTS_FILE_SUFFIX = '.performance-results.json';
|
||||||
* @property {boolean=} ci Run on CI.
|
* @property {boolean=} ci Run on CI.
|
||||||
* @property {number=} rounds Run each test suite this many times for each branch.
|
* @property {number=} rounds Run each test suite this many times for each branch.
|
||||||
* @property {string=} testsBranch The branch whose performance test files will be used for testing.
|
* @property {string=} testsBranch The branch whose performance test files will be used for testing.
|
||||||
|
* @property {boolean=} skipBenchmarking Skip benchmarking and get to report processing (reports supplied from outside).
|
||||||
* @property {string=} wpVersion The WordPress version to be used as the base install for testing.
|
* @property {string=} wpVersion The WordPress version to be used as the base install for testing.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
|
||||||
* A logging helper for printing steps and their substeps.
|
|
||||||
*
|
|
||||||
* @param {number} indent Value to indent the log.
|
|
||||||
* @param {any} msg Message to log.
|
|
||||||
* @param {...any} args Rest of the arguments to pass to console.log.
|
|
||||||
*/
|
|
||||||
function logAtIndent( indent, msg, ...args ) {
|
|
||||||
const prefix = indent === 0 ? '▶ ' : '> ';
|
|
||||||
const newline = indent === 0 ? '\n' : '';
|
|
||||||
return console.log(
|
|
||||||
newline + ' '.repeat( indent ) + prefix + msg,
|
|
||||||
...args
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitizes branch name to be used in a path or a filename.
|
|
||||||
*
|
|
||||||
* @param {string} branch
|
|
||||||
*
|
|
||||||
* @return {string} Sanitized branch name.
|
|
||||||
*/
|
|
||||||
function sanitizeBranchName( branch ) {
|
|
||||||
return branch.replace( /[^a-zA-Z0-9-]/g, '-' );
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Computes the median number from an array numbers.
|
|
||||||
*
|
|
||||||
* @param {number[]} array
|
|
||||||
*
|
|
||||||
* @return {number|undefined} Median value or undefined if array empty.
|
|
||||||
*/
|
|
||||||
function median( array ) {
|
|
||||||
if ( ! array || ! array.length ) return undefined;
|
|
||||||
|
|
||||||
const numbers = [ ...array ].sort( ( a, b ) => a - b );
|
|
||||||
const middleIndex = Math.floor( numbers.length / 2 );
|
|
||||||
|
|
||||||
if ( numbers.length % 2 === 0 ) {
|
|
||||||
return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
|
|
||||||
}
|
|
||||||
return numbers[ middleIndex ];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Runs the performance tests on the current branch.
|
* Runs the performance tests on the current branch.
|
||||||
*
|
*
|
||||||
|
@ -106,6 +62,7 @@ async function runTestSuite( testSuite, testRunnerDir, runKey ) {
|
||||||
*/
|
*/
|
||||||
async function runPerformanceTests( branches, options ) {
|
async function runPerformanceTests( branches, options ) {
|
||||||
const runningInCI = !! process.env.CI || !! options.ci;
|
const runningInCI = !! process.env.CI || !! options.ci;
|
||||||
|
const skipBenchmarking = !! options.skipBenchmarking;
|
||||||
const TEST_ROUNDS = options.rounds || 1;
|
const TEST_ROUNDS = options.rounds || 1;
|
||||||
|
|
||||||
// The default value doesn't work because commander provides an array.
|
// The default value doesn't work because commander provides an array.
|
||||||
|
@ -129,6 +86,20 @@ async function runPerformanceTests( branches, options ) {
|
||||||
await askForConfirmation( 'Ready to go? ' );
|
await askForConfirmation( 'Ready to go? ' );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ( skipBenchmarking ) {
|
||||||
|
// When benchmarking is skipped, it's expected that artifacts folder contains reports for the branches.
|
||||||
|
// If so, we'll process reports and pick test suites as per current state of codebase.
|
||||||
|
const testSuites = getFilesFromDir(
|
||||||
|
path.resolve( __dirname, '../..' ) + config.testsPath
|
||||||
|
).map( ( file ) => {
|
||||||
|
logAtIndent( 1, 'Found:', formats.success( file ) );
|
||||||
|
return path.basename( file, '.spec.js' );
|
||||||
|
} );
|
||||||
|
|
||||||
|
await processPerformanceReports( testSuites, branches );
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
logAtIndent( 0, 'Setting up' );
|
logAtIndent( 0, 'Setting up' );
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -140,7 +111,6 @@ async function runPerformanceTests( branches, options ) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const baseDir = path.join( os.tmpdir(), 'wp-performance-tests' );
|
const baseDir = path.join( os.tmpdir(), 'wp-performance-tests' );
|
||||||
|
|
||||||
if ( fs.existsSync( baseDir ) ) {
|
if ( fs.existsSync( baseDir ) ) {
|
||||||
logAtIndent( 1, 'Removing existing files' );
|
logAtIndent( 1, 'Removing existing files' );
|
||||||
fs.rmSync( baseDir, { recursive: true } );
|
fs.rmSync( baseDir, { recursive: true } );
|
||||||
|
@ -350,82 +320,7 @@ async function runPerformanceTests( branches, options ) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
logAtIndent( 0, 'Calculating results' );
|
await processPerformanceReports( testSuites, branches );
|
||||||
|
|
||||||
const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) =>
|
|
||||||
file.endsWith( RESULTS_FILE_SUFFIX )
|
|
||||||
);
|
|
||||||
/** @type {Record<string,Record<string, Record<string, number>>>} */
|
|
||||||
const results = {};
|
|
||||||
|
|
||||||
// Calculate medians from all rounds.
|
|
||||||
for ( const testSuite of testSuites ) {
|
|
||||||
logAtIndent( 1, 'Test suite:', formats.success( testSuite ) );
|
|
||||||
|
|
||||||
results[ testSuite ] = {};
|
|
||||||
for ( const branch of branches ) {
|
|
||||||
const sanitizedBranchName = sanitizeBranchName( branch );
|
|
||||||
const resultsRounds = resultFiles
|
|
||||||
.filter( ( file ) =>
|
|
||||||
file.includes(
|
|
||||||
`/${ testSuite }_${ sanitizedBranchName }_round-`
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.map( ( file ) => {
|
|
||||||
logAtIndent( 2, 'Reading from:', formats.success( file ) );
|
|
||||||
return readJSONFile( file );
|
|
||||||
} );
|
|
||||||
|
|
||||||
const metrics = Object.keys( resultsRounds[ 0 ] );
|
|
||||||
results[ testSuite ][ branch ] = {};
|
|
||||||
|
|
||||||
for ( const metric of metrics ) {
|
|
||||||
const values = resultsRounds
|
|
||||||
.map( ( round ) => round[ metric ] )
|
|
||||||
.filter( ( value ) => typeof value === 'number' );
|
|
||||||
|
|
||||||
const value = median( values );
|
|
||||||
if ( value !== undefined ) {
|
|
||||||
results[ testSuite ][ branch ][ metric ] = value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const calculatedResultsPath = path.join(
|
|
||||||
ARTIFACTS_PATH,
|
|
||||||
testSuite + RESULTS_FILE_SUFFIX
|
|
||||||
);
|
|
||||||
|
|
||||||
logAtIndent(
|
|
||||||
2,
|
|
||||||
'Saving curated results to:',
|
|
||||||
formats.success( calculatedResultsPath )
|
|
||||||
);
|
|
||||||
fs.writeFileSync(
|
|
||||||
calculatedResultsPath,
|
|
||||||
JSON.stringify( results[ testSuite ], null, 2 )
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
logAtIndent( 0, 'Printing results' );
|
|
||||||
|
|
||||||
for ( const testSuite of testSuites ) {
|
|
||||||
logAtIndent( 0, formats.success( testSuite ) );
|
|
||||||
|
|
||||||
// Invert the results so we can display them in a table.
|
|
||||||
/** @type {Record<string, Record<string, string>>} */
|
|
||||||
const invertedResult = {};
|
|
||||||
for ( const [ branch, metrics ] of Object.entries(
|
|
||||||
results[ testSuite ]
|
|
||||||
) ) {
|
|
||||||
for ( const [ metric, value ] of Object.entries( metrics ) ) {
|
|
||||||
invertedResult[ metric ] = invertedResult[ metric ] || {};
|
|
||||||
invertedResult[ metric ][ branch ] = `${ value } ms`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Print the results.
|
|
||||||
console.table( invertedResult );
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -0,0 +1,119 @@
|
||||||
|
/**
|
||||||
|
* External dependencies
|
||||||
|
*/
|
||||||
|
const bold = require( 'chalk' );
|
||||||
|
const fs = require( 'fs' );
|
||||||
|
const path = require( 'path' );
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Internal dependencies
|
||||||
|
*/
|
||||||
|
const {
|
||||||
|
getFilesFromDir,
|
||||||
|
readJSONFile,
|
||||||
|
logAtIndent,
|
||||||
|
sanitizeBranchName,
|
||||||
|
median
|
||||||
|
} = require( './utils' ) ;
|
||||||
|
|
||||||
|
const formats = {
|
||||||
|
success: bold.green,
|
||||||
|
};
|
||||||
|
|
||||||
|
const ARTIFACTS_PATH =
|
||||||
|
process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
|
||||||
|
const RESULTS_FILE_SUFFIX = '.performance-results.json';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates and prints results from the generated reports.
|
||||||
|
*
|
||||||
|
* @param {string[]} testSuites Test suites we are aiming.
|
||||||
|
* @param {string[]} branches Branches we are aiming.
|
||||||
|
*/
|
||||||
|
async function processPerformanceReports(
|
||||||
|
testSuites: string[],
|
||||||
|
branches: string[]
|
||||||
|
): Promise< void > {
|
||||||
|
logAtIndent( 0, 'Calculating results' );
|
||||||
|
|
||||||
|
const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter(
|
||||||
|
( file: string ) => file.endsWith( RESULTS_FILE_SUFFIX )
|
||||||
|
);
|
||||||
|
const results: Record<
|
||||||
|
string,
|
||||||
|
Record< string, Record< string, number > >
|
||||||
|
> = {};
|
||||||
|
|
||||||
|
// Calculate medians from all rounds.
|
||||||
|
for ( const testSuite of testSuites ) {
|
||||||
|
logAtIndent( 1, 'Test suite:', formats.success( testSuite ) );
|
||||||
|
|
||||||
|
results[ testSuite ] = {};
|
||||||
|
for ( const branch of branches ) {
|
||||||
|
const sanitizedBranchName = sanitizeBranchName( branch );
|
||||||
|
const resultsRounds: any[] = resultFiles
|
||||||
|
.filter( ( file: string ) =>
|
||||||
|
file.includes(
|
||||||
|
`/${ testSuite }_${ sanitizedBranchName }_round-`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.map( ( file: string ) => {
|
||||||
|
logAtIndent( 2, 'Reading from:', formats.success( file ) );
|
||||||
|
return readJSONFile( file );
|
||||||
|
} );
|
||||||
|
|
||||||
|
const metrics = Object.keys( resultsRounds[ 0 ] );
|
||||||
|
results[ testSuite ][ branch ] = {};
|
||||||
|
|
||||||
|
for ( const metric of metrics ) {
|
||||||
|
const values = resultsRounds
|
||||||
|
.map( ( round ) => round[ metric ] )
|
||||||
|
.filter( ( value ) => typeof value === 'number' );
|
||||||
|
|
||||||
|
const value = median( values );
|
||||||
|
if ( value !== undefined ) {
|
||||||
|
results[ testSuite ][ branch ][ metric ] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const calculatedResultsPath = path.join(
|
||||||
|
ARTIFACTS_PATH,
|
||||||
|
testSuite + RESULTS_FILE_SUFFIX
|
||||||
|
);
|
||||||
|
|
||||||
|
logAtIndent(
|
||||||
|
2,
|
||||||
|
'Saving curated results to:',
|
||||||
|
formats.success( calculatedResultsPath )
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
calculatedResultsPath,
|
||||||
|
JSON.stringify( results[ testSuite ], null, 2 )
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
logAtIndent( 0, 'Printing results' );
|
||||||
|
|
||||||
|
for ( const testSuite of testSuites ) {
|
||||||
|
logAtIndent( 0, formats.success( testSuite ) );
|
||||||
|
|
||||||
|
// Invert the results, so we can display them in a table.
|
||||||
|
const invertedResult: Record< string, Record< string, string > > = {};
|
||||||
|
for ( const [ branch, metrics ] of Object.entries(
|
||||||
|
results[ testSuite ]
|
||||||
|
) ) {
|
||||||
|
for ( const [ metric, value ] of Object.entries( metrics ) ) {
|
||||||
|
invertedResult[ metric ] = invertedResult[ metric ] || {};
|
||||||
|
invertedResult[ metric ][ branch ] = `${ value } ms`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Print the results.
|
||||||
|
// eslint-disable-next-line no-console
|
||||||
|
console.table( invertedResult );
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
processPerformanceReports,
|
||||||
|
};
|
|
@ -0,0 +1,15 @@
|
||||||
|
{
|
||||||
|
"extends": "./node_modules/@tsconfig/node16/tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"module": "Node16",
|
||||||
|
"moduleResolution": "Node16",
|
||||||
|
"typeRoots": [
|
||||||
|
"./typings",
|
||||||
|
"./node_modules/@types"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"ts-node": {
|
||||||
|
"transpileOnly": true,
|
||||||
|
"files": true,
|
||||||
|
},
|
||||||
|
}
|
|
@ -97,9 +97,58 @@ function getFilesFromDir( dir ) {
|
||||||
.map( ( dirent ) => path.join( dir, dirent.name ) );
|
.map( ( dirent ) => path.join( dir, dirent.name ) );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A logging helper for printing steps and their substeps.
|
||||||
|
*
|
||||||
|
* @param {number} indent Value to indent the log.
|
||||||
|
* @param {any} msg Message to log.
|
||||||
|
* @param {...any} args Rest of the arguments to pass to console.log.
|
||||||
|
*/
|
||||||
|
function logAtIndent( indent, msg, ...args ) {
|
||||||
|
const prefix = indent === 0 ? '▶ ' : '> ';
|
||||||
|
const newline = indent === 0 ? '\n' : '';
|
||||||
|
return console.log(
|
||||||
|
newline + ' '.repeat( indent ) + prefix + msg,
|
||||||
|
...args
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sanitizes branch name to be used in a path or a filename.
|
||||||
|
*
|
||||||
|
* @param {string} branch
|
||||||
|
*
|
||||||
|
* @return {string} Sanitized branch name.
|
||||||
|
*/
|
||||||
|
function sanitizeBranchName( branch ) {
|
||||||
|
return branch.replace( /[^a-zA-Z0-9-]/g, '-' );
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes the median number from an array numbers.
|
||||||
|
*
|
||||||
|
* @param {number[]} array
|
||||||
|
*
|
||||||
|
* @return {number|undefined} Median value or undefined if array empty.
|
||||||
|
*/
|
||||||
|
function median( array ) {
|
||||||
|
if ( ! array || ! array.length ) return undefined;
|
||||||
|
|
||||||
|
const numbers = [ ...array ].sort( ( a, b ) => a - b );
|
||||||
|
const middleIndex = Math.floor( numbers.length / 2 );
|
||||||
|
|
||||||
|
if ( numbers.length % 2 === 0 ) {
|
||||||
|
return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
|
||||||
|
}
|
||||||
|
return numbers[ middleIndex ];
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
askForConfirmation,
|
askForConfirmation,
|
||||||
readJSONFile,
|
readJSONFile,
|
||||||
runShellScript,
|
runShellScript,
|
||||||
getFilesFromDir,
|
getFilesFromDir,
|
||||||
|
logAtIndent,
|
||||||
|
sanitizeBranchName,
|
||||||
|
median
|
||||||
};
|
};
|
||||||
|
|
Loading…
Reference in New Issue