woocommerce/.github/workflows/scripts/prepare-test-summary.js

220 lines
5.9 KiB
JavaScript
Raw Normal View History

/**
* Script to generate the test results summary to be posted as a GitHub Job Summary and as a PR comment.
*/
const {
API_SUMMARY_PATH,
E2E_PW_SUMMARY_PATH,
SHA,
PR_NUMBER,
E2E_GRAND_TOTAL,
} = process.env;
/**
* Convert the given `duration` from milliseconds to a more user-friendly string.
* For example, if `duration = 323000`, this function would return `5m 23s`.
*
* @param {Number} duration Duration in millisecods, as read from either the `summary.json` file in the Allure report, or from the `test-results.json` file from the Jest-Puppeteer report.
* @returns String in "5m 23s" format.
*/
const getFormattedDuration = ( duration ) => {
const durationMinutes = Math.floor( duration / 1000 / 60 );
const durationSeconds = Math.floor( ( duration / 1000 ) % 60 );
return `${ durationMinutes }m ${ durationSeconds }s`;
};
/**
* Extract the test report statistics (the number of tests that passed, failed, skipped, etc.) from Allure report's `summary.json` file.
*
* @param {string} summaryJSONPath Path to the Allure report's `summary.json` file.
* @returns An object containing relevant statistics from the Allure report.
*/
const getAllureSummaryStats = ( summaryJSONPath ) => {
const summary = require( summaryJSONPath );
const { statistic, time } = summary;
const { passed, failed, skipped, broken, unknown, total } = statistic;
const { duration } = time;
return {
passed,
failed,
skipped,
broken,
unknown,
total,
duration,
};
};
/**
* Construct the array to be used for the API table row.
*
* @returns Array of API test result stats.
*/
const createAPITableRow = () => {
const { passed, failed, skipped, broken, unknown, total, duration } =
getAllureSummaryStats( API_SUMMARY_PATH );
const durationFormatted = getFormattedDuration( duration );
return [
'API Tests',
passed.toString(),
failed.toString(),
broken.toString(),
skipped.toString(),
unknown.toString(),
total.toString(),
durationFormatted,
];
};
/**
* Construct the array to be used for the E2E table row.
*
* @returns Array of E2E test result stats.
*/
const createE2ETableRow = () => {
const { passed, failed, skipped, broken, unknown, total, duration } =
getAllureSummaryStats( E2E_PW_SUMMARY_PATH );
const durationFormatted = getFormattedDuration( duration );
return [
'E2E Tests',
passed.toString(),
failed.toString(),
broken.toString(),
skipped.toString(),
unknown.toString(),
total.toString(),
durationFormatted,
];
};
/**
* Add a warning when the number of executed Playwright E2E tests were fewer than the total.
*/
const addWarningE2EIncomplete = ( warnings ) => {
const { statistic } = require( E2E_PW_SUMMARY_PATH );
const { total } = statistic;
const expectedTotal = Number( E2E_GRAND_TOTAL );
if ( total < expectedTotal ) {
warnings.push(
`INCOMPLETE E2E TEST RUN. We have a total of ${ expectedTotal } E2E tests, but only ${ total } were executed. Note that in CI, E2E tests automatically end when they encounter too many failures.`
);
}
};
/**
*
* Add a warning when there are failures and broken tests.
*/
const addWarningFailuresBrokenTests = ( warnings ) => {
const { failed: apiFailed, broken: apiBroken } =
getAllureSummaryStats( API_SUMMARY_PATH );
const { failed: e2eFailed, broken: e2eBroken } =
getAllureSummaryStats( E2E_PW_SUMMARY_PATH );
if ( apiFailed || apiBroken || e2eFailed || e2eBroken ) {
warnings.push(
'FAILED/BROKEN TESTS. There were failed and/or broken API and E2E tests.'
);
}
};
/**
* Add warnings to the test summary.
*
* @param core The GitHub Actions toolkit core object
*/
const addSummaryWarnings = ( core ) => {
const warnings = [];
addWarningFailuresBrokenTests( warnings );
addWarningE2EIncomplete( warnings );
if ( warnings.length > 0 ) {
core.summary
.addHeading( ':warning: Warning', 3 )
.addRaw(
'Please address the following issues prior to merging this pull request:'
)
.addList( warnings );
}
};
/**
* Create the heading, commit SHA, and test results table.
*
* @param core The GitHub Actions toolkit core object
*/
const addSummaryHeadingAndTable = ( core ) => {
const apiTableRow = createAPITableRow();
const e2eTableRow = createE2ETableRow();
core.summary
.addHeading( 'Test Results Summary' )
.addRaw( `Commit SHA: ${ SHA }` )
.addBreak()
.addBreak()
.addTable( [
[
{ data: 'Test :test_tube:', header: true },
{ data: 'Passed :white_check_mark:', header: true },
{ data: 'Failed :rotating_light:', header: true },
{ data: 'Broken :construction:', header: true },
{ data: 'Skipped :next_track_button:', header: true },
{ data: 'Unknown :grey_question:', header: true },
{ data: 'Total :bar_chart:', header: true },
{ data: 'Duration :stopwatch:', header: true },
],
apiTableRow,
e2eTableRow,
] );
};
/**
* Add the summary footer.
*
* @param core The GitHub Actions toolkit core object
*/
const addSummaryFooter = ( core ) => {
core.summary
.addSeparator()
.addRaw( 'To view the full API test report, click ' )
.addLink(
'here.',
`https://woocommerce.github.io/woocommerce-test-reports/pr/${ PR_NUMBER }/api/`
)
.addBreak()
.addRaw( 'To view the full E2E test report, click ' )
.addLink(
'here.',
`https://woocommerce.github.io/woocommerce-test-reports/pr/${ PR_NUMBER }/e2e/`
)
.addBreak()
.addRaw( 'To view all test reports, visit the ' )
.addLink(
'WooCommerce Test Reports Dashboard.',
'https://woocommerce.github.io/woocommerce-test-reports/'
);
};
/**
* Generate the contents of the test results summary and post it on the workflow run.
*
* @param {*} params Objects passed from the calling GitHub Action workflow.
* @returns Stringified content of the test results summary.
*/
module.exports = async ( { core } ) => {
addSummaryHeadingAndTable( core );
addSummaryWarnings( core );
addSummaryFooter( core );
const summary = core.summary.stringify();
await core.summary.write();
return summary;
};