[testing workflows] Add optional jobs in CI workflow (#47414)

* Include performance tests job in the needed jobs for evaluation

* Use Github api to get the workflow jobs status

* Evaluate project-jobs

* See results

* Check for cancelled jobs

* Add evaluate-project-jobs.js and rename optional jobs

* Fix script path

* Checkout first

* And then fix the path again.

* Update env variable name and be more detailed in what are the required variables

* Use a flat matrix env variable

* Update to trigger test jobs

* Use the jobs name to evaluate optional requirement

* Fix conditions

* Prettier print

* Test lint job failure

* Remove unused MATRIX env variable

* Fix test

* Remove unused MATRIX variable check

* Only run my account e2e tests

* Remove unused function

* Nicer console printing

* Revert change that triggers failing lint job

* Force failing e2e test

* Revert e2e test command and forced failure

* Added test data and test mode

* Added more test data

* Fixed lint errors and warnings

* Exclude .github folder from eslintignore

* Change to trigger everything

* Revert change to trigger everything

.eslintignore

@@ -1 +1,2 @@
 node_modules
+!.github

.github/workflows/ci.yml

@@ -48,7 +48,7 @@ jobs:
                       child_process.execSync( `pnpm utils ci-jobs ${ baseRef } --event ${ githubEvent }` );
 
     project-lint-jobs:
-        name: 'Lint - ${{ matrix.projectName }}'
+        name: "Lint - ${{ matrix.projectName }} ${{ matrix.optional && ' (optional)' || ''}}"
         runs-on: 'ubuntu-20.04'
         needs: 'project-jobs'
         if: ${{ needs.project-jobs.outputs.lint-jobs != '[]' }}
@@ -71,7 +71,7 @@ jobs:
               run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
 
     project-default-test-jobs:
-        name: 'Test - ${{ matrix.projectName }} - ${{ matrix.name }}'
+        name: "Test - ${{ matrix.projectName }} - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || '' || ''}}"
         runs-on: 'ubuntu-20.04'
         needs: 'project-jobs'
         if: ${{ needs.project-jobs.outputs.default-test-jobs != '[]' }}
@@ -97,7 +97,7 @@ jobs:
               run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
 
     project-e2e-test-jobs:
-        name: 'E2E - ${{ matrix.name }}'
+        name: "E2E - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
         runs-on: 'ubuntu-20.04'
         needs: 'project-jobs'
         if: ${{ needs.project-jobs.outputs.e2e-test-jobs != '[]' }}
@@ -153,7 +153,7 @@ jobs:
                   compression-level: 9
 
     project-api-test-jobs:
-        name: 'API - ${{ matrix.name }}'
+        name: "API - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
         runs-on: 'ubuntu-20.04'
         needs: 'project-jobs'
         if: ${{ needs.project-jobs.outputs.api-test-jobs != '[]' }}
@@ -190,7 +190,7 @@ jobs:
                   compression-level: 9
 
     project-performance-test-jobs:
-        name: 'Performance - ${{ matrix.name }}'
+        name: "Performance - ${{ matrix.name }} ${{ matrix.optional && ' (optional)' || ''}}"
         runs-on: 'ubuntu-20.04'
         needs: 'project-jobs'
         if: ${{ needs.project-jobs.outputs.performance-test-jobs != '[]' }}
@@ -244,37 +244,27 @@ jobs:
                 'project-default-test-jobs',
                 'project-e2e-test-jobs',
                 'project-api-test-jobs',
+                'project-performance-test-jobs'
             ]
         if: ${{ always() }}
         steps:
+            - uses: 'actions/checkout@v4'
+              name: 'Checkout'
+              
             - name: 'Evaluation'
+              env:
+                REPOSITORY: ${{ github.repository }}
+                RUN_ID: ${{ github.run_id }}
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
               run: |
+                  # Check if project-jobs was successful. Fail for any other status, including skipped. 
                   result="${{ needs.project-jobs.result }}"
-                  if [[ $result != "success" && $result != "skipped" ]]; then
+                  if [[ $result != "success" ]]; then
-                    echo "An error occurred generating the CI jobs."
+                    echo "Generating CI jobs was not successful."
                     exit 1
                   fi
-                  result="${{ needs.project-lint-jobs.result }}"
+                  
-                  if [[ $result != "success" && $result != "skipped" ]]; then
+                  node .github/workflows/scripts/evaluate-jobs-conclusions.js
-                    echo "One or more lint jobs have failed."
-                    exit 1
-                  fi
-                  result="${{ needs.project-default-test-jobs.result }}"
-                  if [[ $result != "success" && $result != "skipped" ]]; then
-                    echo "One or more test jobs have failed."
-                    exit 1
-                  fi
-                  result="${{ needs.project-e2e-test-jobs.result }}"
-                  if [[ $result != "success" && $result != "skipped" ]]; then
-                    echo "One or more e2e test jobs have failed."
-                    exit 1
-                  fi
-                  result="${{ needs.project-api-test-jobs.result }}"
-                  if [[ $result != "success" && $result != "skipped" ]]; then
-                    echo "One or more api test jobs have failed."
-                    exit 1
-                  fi
-                  echo "All jobs have completed successfully."
 
     e2e-test-reports:
         name: 'Report e2e tests results'

.github/workflows/scripts/evaluate-jobs-conclusions-test-data.json

@@ -0,0 +1,82 @@
+[
+	{
+		"status": "completed",
+		"conclusion": "success",
+		"name": "Successful required job"
+	},
+	{
+		"status": "completed",
+		"conclusion": "success",
+		"name": "Successful job (optional)"
+	},
+	{
+		"status": "completed",
+		"conclusion": "failure",
+		"name": "Failed required job"
+	},
+	{
+		"status": "completed",
+		"conclusion": "failure",
+		"name": "Failed job (optional)"
+	},
+	{
+		"status": "completed",
+		"conclusion": "cancelled",
+		"name": "Cancelled required job"
+	},
+	{
+		"status": "completed",
+		"conclusion": "cancelled",
+		"name": "Cancelled job (optional)"
+	},
+	{
+		"status": "completed",
+		"conclusion": "skipped",
+		"name": "Skipped required job"
+	},
+	{
+		"status": "completed",
+		"conclusion": "skipped",
+		"name": "Skipped job (optional)"
+	},
+	{
+		"status": "queued",
+		"conclusion": "",
+		"name": "Queued required job"
+	},
+	{
+		"status": "queued",
+		"conclusion": "",
+		"name": "Queued job (optional)"
+	},
+	{
+		"status": "in_progress",
+		"conclusion": "",
+		"name": "In progress required job"
+	},
+	{
+		"status": "in_progress",
+		"conclusion": "",
+		"name": "In progress job (optional)"
+	},
+	{
+		"status": "unknown_status",
+		"conclusion": "",
+		"name": "Required job with unknown status"
+	},
+	{
+		"status": "unknown_status",
+		"conclusion": "",
+		"name": "Job with unknown status (optional)"
+	},
+	{
+		"status": "completed",
+		"conclusion": "unknown_conclusion",
+		"name": "Required job with unknown conclusion"
+	},
+	{
+		"status": "completed",
+		"conclusion": "unknown_conclusion",
+		"name": "Job with unknown conclusion (optional)"
+	}
+]

.github/workflows/scripts/evaluate-jobs-conclusions.js

@@ -0,0 +1,91 @@
+/* eslint-disable no-console */
+const { REPOSITORY, RUN_ID, GITHUB_TOKEN, TEST_MODE } = process.env;
+const IGNORED_JOBS = [
+	'Evaluate Project Job Statuses',
+	'Report e2e tests results',
+	'Report API tests results',
+];
+
+const isJobRequired = ( job ) => {
+	return (
+		! job.name.endsWith( '(optional)' ) &&
+		! IGNORED_JOBS.includes( job.name )
+	);
+};
+
+const fetchJobs = async () => {
+	try {
+		const response = await fetch(
+			`https://api.github.com/repos/${ REPOSITORY }/actions/runs/${ RUN_ID }/jobs`,
+			{
+				headers: {
+					'User-Agent': 'node.js',
+					Authorization: `Bearer ${ GITHUB_TOKEN }`,
+				},
+			}
+		);
+		const data = await response.json();
+		return data.jobs;
+	} catch ( error ) {
+		console.error( 'Error:', error );
+		// We want to fail if there is an error getting the jobs conclusions
+		process.exit( 1 );
+	}
+};
+
+const evaluateJobs = async () => {
+	let jobs;
+
+	if ( TEST_MODE ) {
+		jobs = require( './evaluate-jobs-conclusions-test-data.json' );
+	} else {
+		jobs = await fetchJobs();
+	}
+	const nonSuccessfulCompletedJobs = jobs.filter(
+		( job ) =>
+			job.status === 'completed' &&
+			job.conclusion !== 'success' &&
+			job.conclusion !== 'skipped'
+	);
+
+	console.log( 'Workflow jobs:', jobs.length );
+	console.log(
+		'Non successful completed jobs:',
+		nonSuccessfulCompletedJobs.length
+	);
+
+	const failed = [];
+
+	nonSuccessfulCompletedJobs.forEach( ( job ) => {
+		const jobPrintName = `'${ job.name }': ${ job.status }, ${ job.conclusion }`;
+		if ( isJobRequired( job ) ) {
+			console.error( `❌ ${ jobPrintName }, required` );
+			failed.push( job.name );
+		} else {
+			console.warn( `✅ ${ jobPrintName }, optional` );
+		}
+	} );
+
+	if ( failed.length > 0 ) {
+		console.error( 'Failed required jobs:', failed );
+		process.exit( 1 );
+	}
+};
+
+const validateEnvironmentVariables = ( variables ) => {
+	if ( TEST_MODE ) {
+		return;
+	}
+
+	variables.forEach( ( variable ) => {
+		if ( ! process.env[ variable ] ) {
+			console.error( `Missing ${ variable } environment variable` );
+			process.exit( 1 );
+		}
+	} );
+};
+
+validateEnvironmentVariables( [ 'REPOSITORY', 'RUN_ID', 'GITHUB_TOKEN' ] );
+evaluateJobs().then( () => {
+	console.log( 'All required jobs passed' );
+} );