Steve-Dee-Designs
/
woocommerce
mirror of https://github.com/woocommerce/woocommerce.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[dev] Monorepo: introduce reporting mode for perf utility. (#51366) 

Monorepo: introduce reporting mode for perf utility.
This commit is contained in:
Vladimir Reznichenko 2024-09-13 08:58:44 +02:00 committed by GitHub
parent 4dad4a9265
commit 55f855a2e6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 233 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pnpm-lock.yaml
View File

@ -4658,6 +4658,12 @@ importers:
tools/compare-perf:
dependencies:
'@tsconfig/node16':
specifier: ^1.0.4
version: 1.0.4
'@types/node':
specifier: ^16.18.68
version: 16.18.68
'@wordpress/env':
specifier: ^10.1.0
version: 10.5.0
@ -4673,6 +4679,12 @@ importers:
simple-git:
specifier: ^3.21.0
version: 3.21.0
ts-node:
specifier: ^10.9.2
version: 10.9.2(@swc/core@1.3.100)(@types/node@16.18.68)(typescript@5.3.3)
tslib:
specifier: ^2.6.2
version: 2.6.3
tools/monorepo-merge:
dependencies:

11
tools/compare-perf/index.js
View File

@ -15,12 +15,17 @@ const catchException = ( command ) => {
};
};
const ciOption = [ '-c, --ci', 'Run in CI (non interactive)' ];
program
.command( 'compare-performance [branches...]' )
.alias( 'perf' )
.option( ...ciOption )
.option(
'-c, --ci',
'Run in CI (non interactive)'
)
.option(
'--skip-benchmarking',
'Skips benchmarking and gets straight to reporting phase (tests results already available)'
)
.option(
'--rounds <count>',
'Run each test suite this many times for each branch; results are summarized, default = 1'

8
tools/compare-perf/package.json
View File

@ -7,15 +7,19 @@
"license": "GPLv2",
"repository": "woocommerce/woocommerce",
"scripts": {
"compare": "node index.js",
"compare": "node -r ts-node/register index.js",
"log": "node log-to-codevitals.js"
},
"dependencies": {
"@types/node": "^16.18.68",
"@tsconfig/node16": "^1.0.4",
"@wordpress/env": "^10.1.0",
"commander": "9.5.0",
"chalk": "^4.1.2",
"inquirer": "^7.1.0",
"simple-git": "^3.21.0"
"simple-git": "^3.21.0",
"ts-node": "^10.9.2",
"tslib": "^2.6.2"
},
"engines": {
"node": "^20.11.1",

153
tools/compare-perf/performance.js
View File

@ -13,71 +13,27 @@ const formats = {
};
const {
runShellScript,
readJSONFile,
askForConfirmation,
getFilesFromDir,
logAtIndent,
sanitizeBranchName,
} = require( './utils' );
const config = require( './config' );
const { processPerformanceReports } = require( './process-reports.ts' );
const ARTIFACTS_PATH =
process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
const RESULTS_FILE_SUFFIX = '.performance-results.json';
/**
* @typedef WPPerformanceCommandOptions
*
* @property {boolean=} ci Run on CI.
* @property {number=} rounds Run each test suite this many times for each branch.
* @property {string=} testsBranch The branch whose performance test files will be used for testing.
* @property {string=} wpVersion The WordPress version to be used as the base install for testing.
* @property {boolean=} ci Run on CI.
* @property {number=} rounds Run each test suite this many times for each branch.
* @property {string=} testsBranch The branch whose performance test files will be used for testing.
* @property {boolean=} skipBenchmarking Skip benchmarking and get to report processing (reports supplied from outside).
* @property {string=} wpVersion The WordPress version to be used as the base install for testing.
*/
/**
* A logging helper for printing steps and their substeps.
*
* @param {number} indent Value to indent the log.
* @param {any} msg Message to log.
* @param {...any} args Rest of the arguments to pass to console.log.
*/
function logAtIndent( indent, msg, ...args ) {
const prefix = indent === 0 ? '▶ ' : '> ';
const newline = indent === 0 ? '\n' : '';
return console.log(
newline + ' '.repeat( indent ) + prefix + msg,
...args
);
}
/**
* Sanitizes branch name to be used in a path or a filename.
*
* @param {string} branch
*
* @return {string} Sanitized branch name.
*/
function sanitizeBranchName( branch ) {
return branch.replace( /[^a-zA-Z0-9-]/g, '-' );
}
/**
* Computes the median number from an array numbers.
*
* @param {number[]} array
*
* @return {number|undefined} Median value or undefined if array empty.
*/
function median( array ) {
if ( ! array || ! array.length ) return undefined;
const numbers = [ ...array ].sort( ( a, b ) => a - b );
const middleIndex = Math.floor( numbers.length / 2 );
if ( numbers.length % 2 === 0 ) {
return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
}
return numbers[ middleIndex ];
}
/**
* Runs the performance tests on the current branch.
*
@ -106,6 +62,7 @@ async function runTestSuite( testSuite, testRunnerDir, runKey ) {
*/
async function runPerformanceTests( branches, options ) {
const runningInCI = !! process.env.CI || !! options.ci;
const skipBenchmarking = !! options.skipBenchmarking;
const TEST_ROUNDS = options.rounds || 1;
// The default value doesn't work because commander provides an array.
@ -129,6 +86,20 @@ async function runPerformanceTests( branches, options ) {
await askForConfirmation( 'Ready to go? ' );
}
if ( skipBenchmarking ) {
// When benchmarking is skipped, it's expected that artifacts folder contains reports for the branches.
// If so, we'll process reports and pick test suites as per current state of codebase.
const testSuites = getFilesFromDir(
path.resolve( __dirname, '../..' ) + config.testsPath
).map( ( file ) => {
logAtIndent( 1, 'Found:', formats.success( file ) );
return path.basename( file, '.spec.js' );
} );
await processPerformanceReports( testSuites, branches );
return;
}
logAtIndent( 0, 'Setting up' );
/**
@ -140,7 +111,6 @@ async function runPerformanceTests( branches, options ) {
}
const baseDir = path.join( os.tmpdir(), 'wp-performance-tests' );
if ( fs.existsSync( baseDir ) ) {
logAtIndent( 1, 'Removing existing files' );
fs.rmSync( baseDir, { recursive: true } );
@ -350,82 +320,7 @@ async function runPerformanceTests( branches, options ) {
}
}
logAtIndent( 0, 'Calculating results' );
const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) =>
file.endsWith( RESULTS_FILE_SUFFIX )
);
/** @type {Record<string,Record<string, Record<string, number>>>} */
const results = {};
// Calculate medians from all rounds.
for ( const testSuite of testSuites ) {
logAtIndent( 1, 'Test suite:', formats.success( testSuite ) );
results[ testSuite ] = {};
for ( const branch of branches ) {
const sanitizedBranchName = sanitizeBranchName( branch );
const resultsRounds = resultFiles
.filter( ( file ) =>
file.includes(
`/${ testSuite }_${ sanitizedBranchName }_round-`
)
)
.map( ( file ) => {
logAtIndent( 2, 'Reading from:', formats.success( file ) );
return readJSONFile( file );
} );
const metrics = Object.keys( resultsRounds[ 0 ] );
results[ testSuite ][ branch ] = {};
for ( const metric of metrics ) {
const values = resultsRounds
.map( ( round ) => round[ metric ] )
.filter( ( value ) => typeof value === 'number' );
const value = median( values );
if ( value !== undefined ) {
results[ testSuite ][ branch ][ metric ] = value;
}
}
}
const calculatedResultsPath = path.join(
ARTIFACTS_PATH,
testSuite + RESULTS_FILE_SUFFIX
);
logAtIndent(
2,
'Saving curated results to:',
formats.success( calculatedResultsPath )
);
fs.writeFileSync(
calculatedResultsPath,
JSON.stringify( results[ testSuite ], null, 2 )
);
}
logAtIndent( 0, 'Printing results' );
for ( const testSuite of testSuites ) {
logAtIndent( 0, formats.success( testSuite ) );
// Invert the results so we can display them in a table.
/** @type {Record<string, Record<string, string>>} */
const invertedResult = {};
for ( const [ branch, metrics ] of Object.entries(
results[ testSuite ]
) ) {
for ( const [ metric, value ] of Object.entries( metrics ) ) {
invertedResult[ metric ] = invertedResult[ metric ] || {};
invertedResult[ metric ][ branch ] = `${ value } ms`;
}
}
// Print the results.
console.table( invertedResult );
}
await processPerformanceReports( testSuites, branches );
}
module.exports = {

119
tools/compare-perf/process-reports.ts Normal file
View File

@ -0,0 +1,119 @@
/**
* External dependencies
*/
const bold = require( 'chalk' );
const fs = require( 'fs' );
const path = require( 'path' );
/**
* Internal dependencies
*/
const {
getFilesFromDir,
readJSONFile,
logAtIndent,
sanitizeBranchName,
median
} = require( './utils' ) ;
const formats = {
success: bold.green,
};
const ARTIFACTS_PATH =
process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
const RESULTS_FILE_SUFFIX = '.performance-results.json';
/**
* Calculates and prints results from the generated reports.
*
* @param {string[]} testSuites Test suites we are aiming.
* @param {string[]} branches Branches we are aiming.
*/
async function processPerformanceReports(
testSuites: string[],
branches: string[]
): Promise< void > {
logAtIndent( 0, 'Calculating results' );
const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter(
( file: string ) => file.endsWith( RESULTS_FILE_SUFFIX )
);
const results: Record<
string,
Record< string, Record< string, number > >
> = {};
// Calculate medians from all rounds.
for ( const testSuite of testSuites ) {
logAtIndent( 1, 'Test suite:', formats.success( testSuite ) );
results[ testSuite ] = {};
for ( const branch of branches ) {
const sanitizedBranchName = sanitizeBranchName( branch );
const resultsRounds: any[] = resultFiles
.filter( ( file: string ) =>
file.includes(
`/${ testSuite }_${ sanitizedBranchName }_round-`
)
)
.map( ( file: string ) => {
logAtIndent( 2, 'Reading from:', formats.success( file ) );
return readJSONFile( file );
} );
const metrics = Object.keys( resultsRounds[ 0 ] );
results[ testSuite ][ branch ] = {};
for ( const metric of metrics ) {
const values = resultsRounds
.map( ( round ) => round[ metric ] )
.filter( ( value ) => typeof value === 'number' );
const value = median( values );
if ( value !== undefined ) {
results[ testSuite ][ branch ][ metric ] = value;
}
}
}
const calculatedResultsPath = path.join(
ARTIFACTS_PATH,
testSuite + RESULTS_FILE_SUFFIX
);
logAtIndent(
2,
'Saving curated results to:',
formats.success( calculatedResultsPath )
);
fs.writeFileSync(
calculatedResultsPath,
JSON.stringify( results[ testSuite ], null, 2 )
);
}
logAtIndent( 0, 'Printing results' );
for ( const testSuite of testSuites ) {
logAtIndent( 0, formats.success( testSuite ) );
// Invert the results, so we can display them in a table.
const invertedResult: Record< string, Record< string, string > > = {};
for ( const [ branch, metrics ] of Object.entries(
results[ testSuite ]
) ) {
for ( const [ metric, value ] of Object.entries( metrics ) ) {
invertedResult[ metric ] = invertedResult[ metric ] || {};
invertedResult[ metric ][ branch ] = `${ value } ms`;
}
}
// Print the results.
// eslint-disable-next-line no-console
console.table( invertedResult );
}
}
module.exports = {
processPerformanceReports,
};

15
tools/compare-perf/tsconfig.json Normal file
View File

@ -0,0 +1,15 @@
{
"extends": "./node_modules/@tsconfig/node16/tsconfig.json",
"compilerOptions": {
"module": "Node16",
"moduleResolution": "Node16",
"typeRoots": [
"./typings",
"./node_modules/@types"
]
},
"ts-node": {
"transpileOnly": true,
"files": true,
},
}

49
tools/compare-perf/utils.js
View File

@ -97,9 +97,58 @@ function getFilesFromDir( dir ) {
.map( ( dirent ) => path.join( dir, dirent.name ) );
}
/**
* A logging helper for printing steps and their substeps.
*
* @param {number} indent Value to indent the log.
* @param {any} msg Message to log.
* @param {...any} args Rest of the arguments to pass to console.log.
*/
function logAtIndent( indent, msg, ...args ) {
const prefix = indent === 0 ? '▶ ' : '> ';
const newline = indent === 0 ? '\n' : '';
return console.log(
newline + ' '.repeat( indent ) + prefix + msg,
...args
);
}
/**
* Sanitizes branch name to be used in a path or a filename.
*
* @param {string} branch
*
* @return {string} Sanitized branch name.
*/
function sanitizeBranchName( branch ) {
return branch.replace( /[^a-zA-Z0-9-]/g, '-' );
}
/**
* Computes the median number from an array numbers.
*
* @param {number[]} array
*
* @return {number|undefined} Median value or undefined if array empty.
*/
function median( array ) {
if ( ! array || ! array.length ) return undefined;
const numbers = [ ...array ].sort( ( a, b ) => a - b );
const middleIndex = Math.floor( numbers.length / 2 );
if ( numbers.length % 2 === 0 ) {
return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
}
return numbers[ middleIndex ];
}
module.exports = {
askForConfirmation,
readJSONFile,
runShellScript,
getFilesFromDir,
logAtIndent,
sanitizeBranchName,
median
};