[testing workflows] Add optional jobs in CI workflow (#47414)

* Include performance tests job in the needed jobs for evaluation

* Use Github api to get the workflow jobs status

* Evaluate project-jobs

* See results

* Check for cancelled jobs

* Add evaluate-project-jobs.js and rename optional jobs

* Fix script path

* Checkout first

* And then fix the path again.

* Update env variable name and be more detailed in what are the required variables

* Use a flat matrix env variable

* Update to trigger test jobs

* Use the jobs name to evaluate optional requirement

* Fix conditions

* Prettier print

* Test lint job failure

* Remove unused MATRIX env variable

* Fix test

* Remove unused MATRIX variable check

* Only run my account e2e tests

* Remove unused function

* Nicer console printing

* Revert change that triggers failing lint job

* Force failing e2e test

* Revert e2e test command and forced failure

* Added test data and test mode

* Added more test data

* Fixed lint errors and warnings

* Exclude .github folder from eslintignore

* Change to trigger everything

* Revert change to trigger everything
This commit is contained in:
Adrian Moldovan 2024-05-16 23:31:17 +03:00 committed by GitHub
parent 0a5ab942bd
commit 9b8063f6b2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 193 additions and 29 deletions

View File

@ -1 +1,2 @@
node_modules node_modules
!.github

View File

@ -48,7 +48,7 @@ jobs:
child_process.execSync( `pnpm utils ci-jobs ${ baseRef } --event ${ githubEvent }` ); child_process.execSync( `pnpm utils ci-jobs ${ baseRef } --event ${ githubEvent }` );
project-lint-jobs: project-lint-jobs:
name: 'Lint - ${{ matrix.projectName }}' name: "Lint - ${{ matrix.projectName }} ${{ matrix.optional && ' (optional)' || ''}}"
runs-on: 'ubuntu-20.04' runs-on: 'ubuntu-20.04'
needs: 'project-jobs' needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.lint-jobs != '[]' }} if: ${{ needs.project-jobs.outputs.lint-jobs != '[]' }}
@ -71,7 +71,7 @@ jobs:
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}' run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
project-default-test-jobs: project-default-test-jobs:
name: 'Test - ${{ matrix.projectName }} - ${{ matrix.name }}' name: "Test - ${{ matrix.projectName }} - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || '' || ''}}"
runs-on: 'ubuntu-20.04' runs-on: 'ubuntu-20.04'
needs: 'project-jobs' needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.default-test-jobs != '[]' }} if: ${{ needs.project-jobs.outputs.default-test-jobs != '[]' }}
@ -97,7 +97,7 @@ jobs:
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}' run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
project-e2e-test-jobs: project-e2e-test-jobs:
name: 'E2E - ${{ matrix.name }}' name: "E2E - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
runs-on: 'ubuntu-20.04' runs-on: 'ubuntu-20.04'
needs: 'project-jobs' needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.e2e-test-jobs != '[]' }} if: ${{ needs.project-jobs.outputs.e2e-test-jobs != '[]' }}
@ -153,7 +153,7 @@ jobs:
compression-level: 9 compression-level: 9
project-api-test-jobs: project-api-test-jobs:
name: 'API - ${{ matrix.name }}' name: "API - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
runs-on: 'ubuntu-20.04' runs-on: 'ubuntu-20.04'
needs: 'project-jobs' needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.api-test-jobs != '[]' }} if: ${{ needs.project-jobs.outputs.api-test-jobs != '[]' }}
@ -190,7 +190,7 @@ jobs:
compression-level: 9 compression-level: 9
project-performance-test-jobs: project-performance-test-jobs:
name: 'Performance - ${{ matrix.name }}' name: "Performance - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
runs-on: 'ubuntu-20.04' runs-on: 'ubuntu-20.04'
needs: 'project-jobs' needs: 'project-jobs'
if: ${{ needs.project-jobs.outputs.performance-test-jobs != '[]' }} if: ${{ needs.project-jobs.outputs.performance-test-jobs != '[]' }}
@ -244,37 +244,27 @@ jobs:
'project-default-test-jobs', 'project-default-test-jobs',
'project-e2e-test-jobs', 'project-e2e-test-jobs',
'project-api-test-jobs', 'project-api-test-jobs',
'project-performance-test-jobs'
] ]
if: ${{ always() }} if: ${{ always() }}
steps: steps:
- uses: 'actions/checkout@v4'
name: 'Checkout'
- name: 'Evaluation' - name: 'Evaluation'
env:
REPOSITORY: ${{ github.repository }}
RUN_ID: ${{ github.run_id }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: | run: |
# Check if project-jobs was successful. Fail for any other status, including skipped.
result="${{ needs.project-jobs.result }}" result="${{ needs.project-jobs.result }}"
if [[ $result != "success" && $result != "skipped" ]]; then if [[ $result != "success" ]]; then
echo "An error occurred generating the CI jobs." echo "Generating CI jobs was not successful."
exit 1 exit 1
fi fi
result="${{ needs.project-lint-jobs.result }}"
if [[ $result != "success" && $result != "skipped" ]]; then node .github/workflows/scripts/evaluate-jobs-conclusions.js
echo "One or more lint jobs have failed."
exit 1
fi
result="${{ needs.project-default-test-jobs.result }}"
if [[ $result != "success" && $result != "skipped" ]]; then
echo "One or more test jobs have failed."
exit 1
fi
result="${{ needs.project-e2e-test-jobs.result }}"
if [[ $result != "success" && $result != "skipped" ]]; then
echo "One or more e2e test jobs have failed."
exit 1
fi
result="${{ needs.project-api-test-jobs.result }}"
if [[ $result != "success" && $result != "skipped" ]]; then
echo "One or more api test jobs have failed."
exit 1
fi
echo "All jobs have completed successfully."
e2e-test-reports: e2e-test-reports:
name: 'Report e2e tests results' name: 'Report e2e tests results'

View File

@ -0,0 +1,82 @@
[
{
"status": "completed",
"conclusion": "success",
"name": "Successful required job"
},
{
"status": "completed",
"conclusion": "success",
"name": "Successful job (optional)"
},
{
"status": "completed",
"conclusion": "failure",
"name": "Failed required job"
},
{
"status": "completed",
"conclusion": "failure",
"name": "Failed job (optional)"
},
{
"status": "completed",
"conclusion": "cancelled",
"name": "Cancelled required job"
},
{
"status": "completed",
"conclusion": "cancelled",
"name": "Cancelled job (optional)"
},
{
"status": "completed",
"conclusion": "skipped",
"name": "Skipped required job"
},
{
"status": "completed",
"conclusion": "skipped",
"name": "Skipped job (optional)"
},
{
"status": "queued",
"conclusion": "",
"name": "Queued required job"
},
{
"status": "queued",
"conclusion": "",
"name": "Queued job (optional)"
},
{
"status": "in_progress",
"conclusion": "",
"name": "In progress required job"
},
{
"status": "in_progress",
"conclusion": "",
"name": "In progress job (optional)"
},
{
"status": "unknown_status",
"conclusion": "",
"name": "Required job with unknown status"
},
{
"status": "unknown_status",
"conclusion": "",
"name": "Job with unknown status (optional)"
},
{
"status": "completed",
"conclusion": "unknown_conclusion",
"name": "Required job with unknown conclusion"
},
{
"status": "completed",
"conclusion": "unknown_conclusion",
"name": "Job with unknown conclusion (optional)"
}
]

View File

@ -0,0 +1,91 @@
/* eslint-disable no-console */
const { REPOSITORY, RUN_ID, GITHUB_TOKEN, TEST_MODE } = process.env;
const IGNORED_JOBS = [
'Evaluate Project Job Statuses',
'Report e2e tests results',
'Report API tests results',
];
const isJobRequired = ( job ) => {
return (
! job.name.endsWith( '(optional)' ) &&
! IGNORED_JOBS.includes( job.name )
);
};
const fetchJobs = async () => {
try {
const response = await fetch(
`https://api.github.com/repos/${ REPOSITORY }/actions/runs/${ RUN_ID }/jobs`,
{
headers: {
'User-Agent': 'node.js',
Authorization: `Bearer ${ GITHUB_TOKEN }`,
},
}
);
const data = await response.json();
return data.jobs;
} catch ( error ) {
console.error( 'Error:', error );
// We want to fail if there is an error getting the jobs conclusions
process.exit( 1 );
}
};
const evaluateJobs = async () => {
let jobs;
if ( TEST_MODE ) {
jobs = require( './evaluate-jobs-conclusions-test-data.json' );
} else {
jobs = await fetchJobs();
}
const nonSuccessfulCompletedJobs = jobs.filter(
( job ) =>
job.status === 'completed' &&
job.conclusion !== 'success' &&
job.conclusion !== 'skipped'
);
console.log( 'Workflow jobs:', jobs.length );
console.log(
'Non successful completed jobs:',
nonSuccessfulCompletedJobs.length
);
const failed = [];
nonSuccessfulCompletedJobs.forEach( ( job ) => {
const jobPrintName = `'${ job.name }': ${ job.status }, ${ job.conclusion }`;
if ( isJobRequired( job ) ) {
console.error( `${ jobPrintName }, required` );
failed.push( job.name );
} else {
console.warn( `${ jobPrintName }, optional` );
}
} );
if ( failed.length > 0 ) {
console.error( 'Failed required jobs:', failed );
process.exit( 1 );
}
};
const validateEnvironmentVariables = ( variables ) => {
if ( TEST_MODE ) {
return;
}
variables.forEach( ( variable ) => {
if ( ! process.env[ variable ] ) {
console.error( `Missing ${ variable } environment variable` );
process.exit( 1 );
}
} );
};
validateEnvironmentVariables( [ 'REPOSITORY', 'RUN_ID', 'GITHUB_TOKEN' ] );
evaluateJobs().then( () => {
console.log( 'All required jobs passed' );
} );