[testing workflows] Add optional jobs in CI workflow (#47414)
* Include performance tests job in the needed jobs for evaluation * Use Github api to get the workflow jobs status * Evaluate project-jobs * See results * Check for cancelled jobs * Add evaluate-project-jobs.js and rename optional jobs * Fix script path * Checkout first * And then fix the path again. * Update env variable name and be more detailed in what are the required variables * Use a flat matrix env variable * Update to trigger test jobs * Use the jobs name to evaluate optional requirement * Fix conditions * Prettier print * Test lint job failure * Remove unused MATRIX env variable * Fix test * Remove unused MATRIX variable check * Only run my account e2e tests * Remove unused function * Nicer console printing * Revert change that triggers failing lint job * Force failing e2e test * Revert e2e test command and forced failure * Added test data and test mode * Added more test data * Fixed lint errors and warnings * Exclude .github folder from eslintignore * Change to trigger everything * Revert change to trigger everything
This commit is contained in:
Adrian Moldovan
GitHub
parent
0a5ab942bd
commit
9b8063f6b2
|
|
@ -1 +1,2 @@
|
|||
node_modules
|
||||
node_modules
|
||||
!.github
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ jobs:
|
|||
child_process.execSync( `pnpm utils ci-jobs ${ baseRef } --event ${ githubEvent }` );
|
||||
|
||||
project-lint-jobs:
|
||||
name: 'Lint - ${{ matrix.projectName }}'
|
||||
name: "Lint - ${{ matrix.projectName }} ${{ matrix.optional && ' (optional)' || ''}}"
|
||||
runs-on: 'ubuntu-20.04'
|
||||
needs: 'project-jobs'
|
||||
if: ${{ needs.project-jobs.outputs.lint-jobs != '[]' }}
|
||||
|
|
@ -71,7 +71,7 @@ jobs:
|
|||
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
|
||||
|
||||
project-default-test-jobs:
|
||||
name: 'Test - ${{ matrix.projectName }} - ${{ matrix.name }}'
|
||||
name: "Test - ${{ matrix.projectName }} - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || '' || ''}}"
|
||||
runs-on: 'ubuntu-20.04'
|
||||
needs: 'project-jobs'
|
||||
if: ${{ needs.project-jobs.outputs.default-test-jobs != '[]' }}
|
||||
|
|
@ -97,7 +97,7 @@ jobs:
|
|||
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
|
||||
|
||||
project-e2e-test-jobs:
|
||||
name: 'E2E - ${{ matrix.name }}'
|
||||
name: "E2E - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
|
||||
runs-on: 'ubuntu-20.04'
|
||||
needs: 'project-jobs'
|
||||
if: ${{ needs.project-jobs.outputs.e2e-test-jobs != '[]' }}
|
||||
|
|
@ -153,7 +153,7 @@ jobs:
|
|||
compression-level: 9
|
||||
|
||||
project-api-test-jobs:
|
||||
name: 'API - ${{ matrix.name }}'
|
||||
name: "API - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
|
||||
runs-on: 'ubuntu-20.04'
|
||||
needs: 'project-jobs'
|
||||
if: ${{ needs.project-jobs.outputs.api-test-jobs != '[]' }}
|
||||
|
|
@ -190,7 +190,7 @@ jobs:
|
|||
compression-level: 9
|
||||
|
||||
project-performance-test-jobs:
|
||||
name: 'Performance - ${{ matrix.name }}'
|
||||
name: "Performance - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
|
||||
runs-on: 'ubuntu-20.04'
|
||||
needs: 'project-jobs'
|
||||
if: ${{ needs.project-jobs.outputs.performance-test-jobs != '[]' }}
|
||||
|
|
@ -244,37 +244,27 @@ jobs:
|
|||
'project-default-test-jobs',
|
||||
'project-e2e-test-jobs',
|
||||
'project-api-test-jobs',
|
||||
'project-performance-test-jobs'
|
||||
]
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- uses: 'actions/checkout@v4'
|
||||
name: 'Checkout'
|
||||
|
||||
- name: 'Evaluation'
|
||||
env:
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Check if project-jobs was successful. Fail for any other status, including skipped.
|
||||
result="${{ needs.project-jobs.result }}"
|
||||
if [[ $result != "success" && $result != "skipped" ]]; then
|
||||
echo "An error occurred generating the CI jobs."
|
||||
if [[ $result != "success" ]]; then
|
||||
echo "Generating CI jobs was not successful."
|
||||
exit 1
|
||||
fi
|
||||
result="${{ needs.project-lint-jobs.result }}"
|
||||
if [[ $result != "success" && $result != "skipped" ]]; then
|
||||
echo "One or more lint jobs have failed."
|
||||
exit 1
|
||||
fi
|
||||
result="${{ needs.project-default-test-jobs.result }}"
|
||||
if [[ $result != "success" && $result != "skipped" ]]; then
|
||||
echo "One or more test jobs have failed."
|
||||
exit 1
|
||||
fi
|
||||
result="${{ needs.project-e2e-test-jobs.result }}"
|
||||
if [[ $result != "success" && $result != "skipped" ]]; then
|
||||
echo "One or more e2e test jobs have failed."
|
||||
exit 1
|
||||
fi
|
||||
result="${{ needs.project-api-test-jobs.result }}"
|
||||
if [[ $result != "success" && $result != "skipped" ]]; then
|
||||
echo "One or more api test jobs have failed."
|
||||
exit 1
|
||||
fi
|
||||
echo "All jobs have completed successfully."
|
||||
|
||||
node .github/workflows/scripts/evaluate-jobs-conclusions.js
|
||||
|
||||
e2e-test-reports:
|
||||
name: 'Report e2e tests results'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
[
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "success",
|
||||
"name": "Successful required job"
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "success",
|
||||
"name": "Successful job (optional)"
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "failure",
|
||||
"name": "Failed required job"
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "failure",
|
||||
"name": "Failed job (optional)"
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "cancelled",
|
||||
"name": "Cancelled required job"
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "cancelled",
|
||||
"name": "Cancelled job (optional)"
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "skipped",
|
||||
"name": "Skipped required job"
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "skipped",
|
||||
"name": "Skipped job (optional)"
|
||||
},
|
||||
{
|
||||
"status": "queued",
|
||||
"conclusion": "",
|
||||
"name": "Queued required job"
|
||||
},
|
||||
{
|
||||
"status": "queued",
|
||||
"conclusion": "",
|
||||
"name": "Queued job (optional)"
|
||||
},
|
||||
{
|
||||
"status": "in_progress",
|
||||
"conclusion": "",
|
||||
"name": "In progress required job"
|
||||
},
|
||||
{
|
||||
"status": "in_progress",
|
||||
"conclusion": "",
|
||||
"name": "In progress job (optional)"
|
||||
},
|
||||
{
|
||||
"status": "unknown_status",
|
||||
"conclusion": "",
|
||||
"name": "Required job with unknown status"
|
||||
},
|
||||
{
|
||||
"status": "unknown_status",
|
||||
"conclusion": "",
|
||||
"name": "Job with unknown status (optional)"
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "unknown_conclusion",
|
||||
"name": "Required job with unknown conclusion"
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"conclusion": "unknown_conclusion",
|
||||
"name": "Job with unknown conclusion (optional)"
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
/* eslint-disable no-console */
|
||||
const { REPOSITORY, RUN_ID, GITHUB_TOKEN, TEST_MODE } = process.env;
|
||||
const IGNORED_JOBS = [
|
||||
'Evaluate Project Job Statuses',
|
||||
'Report e2e tests results',
|
||||
'Report API tests results',
|
||||
];
|
||||
|
||||
const isJobRequired = ( job ) => {
|
||||
return (
|
||||
! job.name.endsWith( '(optional)' ) &&
|
||||
! IGNORED_JOBS.includes( job.name )
|
||||
);
|
||||
};
|
||||
|
||||
const fetchJobs = async () => {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`https://api.github.com/repos/${ REPOSITORY }/actions/runs/${ RUN_ID }/jobs`,
|
||||
{
|
||||
headers: {
|
||||
'User-Agent': 'node.js',
|
||||
Authorization: `Bearer ${ GITHUB_TOKEN }`,
|
||||
},
|
||||
}
|
||||
);
|
||||
const data = await response.json();
|
||||
return data.jobs;
|
||||
} catch ( error ) {
|
||||
console.error( 'Error:', error );
|
||||
// We want to fail if there is an error getting the jobs conclusions
|
||||
process.exit( 1 );
|
||||
}
|
||||
};
|
||||
|
||||
const evaluateJobs = async () => {
|
||||
let jobs;
|
||||
|
||||
if ( TEST_MODE ) {
|
||||
jobs = require( './evaluate-jobs-conclusions-test-data.json' );
|
||||
} else {
|
||||
jobs = await fetchJobs();
|
||||
}
|
||||
const nonSuccessfulCompletedJobs = jobs.filter(
|
||||
( job ) =>
|
||||
job.status === 'completed' &&
|
||||
job.conclusion !== 'success' &&
|
||||
job.conclusion !== 'skipped'
|
||||
);
|
||||
|
||||
console.log( 'Workflow jobs:', jobs.length );
|
||||
console.log(
|
||||
'Non successful completed jobs:',
|
||||
nonSuccessfulCompletedJobs.length
|
||||
);
|
||||
|
||||
const failed = [];
|
||||
|
||||
nonSuccessfulCompletedJobs.forEach( ( job ) => {
|
||||
const jobPrintName = `'${ job.name }': ${ job.status }, ${ job.conclusion }`;
|
||||
if ( isJobRequired( job ) ) {
|
||||
console.error( `❌ ${ jobPrintName }, required` );
|
||||
failed.push( job.name );
|
||||
} else {
|
||||
console.warn( `✅ ${ jobPrintName }, optional` );
|
||||
}
|
||||
} );
|
||||
|
||||
if ( failed.length > 0 ) {
|
||||
console.error( 'Failed required jobs:', failed );
|
||||
process.exit( 1 );
|
||||
}
|
||||
};
|
||||
|
||||
const validateEnvironmentVariables = ( variables ) => {
|
||||
if ( TEST_MODE ) {
|
||||
return;
|
||||
}
|
||||
|
||||
variables.forEach( ( variable ) => {
|
||||
if ( ! process.env[ variable ] ) {
|
||||
console.error( `Missing ${ variable } environment variable` );
|
||||
process.exit( 1 );
|
||||
}
|
||||
} );
|
||||
};
|
||||
|
||||
validateEnvironmentVariables( [ 'REPOSITORY', 'RUN_ID', 'GITHUB_TOKEN' ] );
|
||||
evaluateJobs().then( () => {
|
||||
console.log( 'All required jobs passed' );
|
||||
} );
|
||||
Loading…
Reference in New Issue