Add the slack-test-report util (#47805)

* Add the slack-test-report util

* Error if the required GitHub context variables are not set

* Error if the required SLACK_CHANNEL env var is not set

* Use a more generic message to be able to report on jobs that are not tests, like build

* Add the alert-on-failure job in ci

* Run for pull_request to test the job

* Setup monorepo first

* Force a linting error

* Better logging

* Use inputs.trigger as report name if set

* Force an e2e test failure

* Set the commit message in the notification

* Use INPUT_TRIGGER for all notifications

* Revert test changes

.github/workflows/ci.yml

@@ -281,6 +281,45 @@ jobs:
                   
                   node .github/workflows/scripts/evaluate-jobs-conclusions.js
 
+
+    alert-on-failure:
+        name: 'Report results on Slack'
+        runs-on: 'ubuntu-20.04'
+        needs:
+            [
+                'project-jobs',
+                'project-lint-jobs',
+                'project-default-test-jobs',
+                'project-e2e-test-jobs',
+                'project-api-test-jobs',
+                'project-performance-test-jobs'
+            ]
+        if: ${{ always() && github.event_name != 'pull_request'  }}
+        steps:
+            - uses: 'actions/checkout@v4'
+              name: 'Checkout'
+              
+            - uses: './.github/actions/setup-woocommerce-monorepo'
+              name: 'Setup Monorepo'
+              with:
+                php-version: false
+              
+            - name: 'Send messages for failed jobs'
+              env:
+                SLACK_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }}
+                SLACK_CHANNEL: ${{ secrets.TEST_REPORTS_SLACK_CHANNEL }}
+                HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
+                INPUT_TRIGGER: ${{ inputs.trigger }}
+              run: |
+                  COMMIT_MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
+                
+                  pnpm utils slack-test-report -c "${{ needs.project-jobs.result }}" -r "$INPUT_TRIGGER Build jobs matrix" -m "$COMMIT_MESSAGE"
+                  pnpm utils slack-test-report -c "${{ needs.project-lint-jobs.result }}" -r "$INPUT_TRIGGER Linting" -m "$COMMIT_MESSAGE"
+                  pnpm utils slack-test-report -c "${{ needs.project-default-test-jobs.result }}" -r "$INPUT_TRIGGER Tests" -m "$COMMIT_MESSAGE"
+                  pnpm utils slack-test-report -c "${{ needs.project-e2e-test-jobs.result }}" -r "$INPUT_TRIGGER e2e tests" -m "$COMMIT_MESSAGE"
+                  pnpm utils slack-test-report -c "${{ needs.project-api-test-jobs.result }}" -r "$INPUT_TRIGGER Api tests" -m "$COMMIT_MESSAGE"
+                  pnpm utils slack-test-report -c "${{ needs.project-performance-test-jobs.result }}" -r "$INPUT_TRIGGER Performance tests" -m "$COMMIT_MESSAGE"
+
     e2e-test-reports:
         name: 'Report e2e tests results'
         needs: [project-e2e-test-jobs]
@@ -340,16 +379,7 @@ jobs:
                   else
                     echo "No report will be created for '$GITHUB_EVENT_NAME' event"
                   fi
-
+                
-            - name: 'Send Slack notification'
-              if: ${{ always() && github.event_name != 'pull_request' }}
-              uses: automattic/action-test-results-to-slack@v0.3.0
-              with:
-                  github_token: ${{ secrets.GITHUB_TOKEN }}
-                  slack_token: ${{ secrets.E2E_SLACK_TOKEN }}
-                  slack_channel: ${{ secrets.TEST_REPORTS_SLACK_CHANNEL }}
-                  playwright_report_path: ./out/test-results-*.json
-                  playwright_output_dir: ./out/results-data
 
     api-test-reports:
         name: 'Report API tests results'
@@ -410,11 +440,3 @@ jobs:
                   else
                     echo "No report will be created for '$GITHUB_EVENT_NAME' event"
                   fi
-
-            - name: 'Send Slack notification'
-              if: github.event_name != 'pull_request'
-              uses: automattic/action-test-results-to-slack@v0.3.0
-              with:
-                  github_token: ${{ secrets.GITHUB_TOKEN }}
-                  slack_token: ${{ secrets.E2E_SLACK_TOKEN }}
-                  slack_channel: ${{ secrets.TEST_REPORTS_SLACK_CHANNEL }}

tools/monorepo-utils/src/index.ts

@@ -15,6 +15,7 @@ import Manifest from './md-docs/commands';
 import Changefile from './changefile';
 import CIJobs from './ci-jobs';
 import WorkflowProfiler from './workflow-profiler/commands';
+import SlackTestReport from './slack-test-report';
 import { Logger } from './core/logger';
 import { isGithubCI } from './core/environment';
 
@@ -36,7 +37,8 @@ const program = new Command()
 	.addCommand( Changefile )
 	.addCommand( CIJobs )
 	.addCommand( WorkflowProfiler )
-	.addCommand( Manifest );
+	.addCommand( Manifest )
+	.addCommand( SlackTestReport );
 
 program.exitOverride();
 

tools/monorepo-utils/src/slack-test-report/README.md

@@ -0,0 +1,5 @@
+# Slack Test Report
+
+Send a test report to Slack.
+
+To see available commands run `pnpm utils slack-test-report --help` from the project root.

tools/monorepo-utils/src/slack-test-report/index.ts

@@ -0,0 +1,107 @@
+/**
+ * External dependencies
+ */
+import { Command } from '@commander-js/extra-typings';
+import { WebClient } from '@slack/web-api';
+
+/**
+ * Internal dependencies
+ */
+import { Logger } from '../core/logger';
+import { getEnvVar } from '../core/environment';
+import { createMessage, postMessage } from './lib/message';
+
+const conclusions = [ 'success', 'failure', 'skipped', 'cancelled' ];
+
+const program = new Command( 'slack-test-report' )
+	.description( 'Send a test report to Slack' )
+	.requiredOption(
+		'-c --conclusion <conclusion>',
+		`Test run conclusion. Expected one of: ${ conclusions }`
+	)
+	.option(
+		'-r --report-name <reportName>',
+		'The name of the report. Example: "post-merge tests", "daily e2e tests"',
+		''
+	)
+	.option(
+		'-u --username <username>',
+		'The Slack username.',
+		'Github reporter'
+	)
+	.option(
+		'-n --pr-number <prNumber>',
+		'The PR number to be included in the message, if the event is pull_request.',
+		''
+	)
+	.option(
+		'-t --pr-title <prTitle>',
+		'The PR title to be included in the message, if the event is pull_request.',
+		'Default PR title'
+	)
+	.option( '-m --commit-message <commitMessage>', 'The commit message.', '' )
+	.action( async ( options ) => {
+		if ( options.reportName === '' ) {
+			Logger.warn(
+				'No report name was specified. Using a default message.'
+			);
+		}
+
+		const isFailure = options.conclusion === 'failure';
+		const channel = getEnvVar( 'SLACK_CHANNEL', true );
+
+		if ( isFailure ) {
+			const { username } = options;
+			const client = new WebClient( getEnvVar( 'SLACK_TOKEN', true ) );
+			const { text, mainMsgBlocks, detailsMsgBlocksChunks } =
+				await createMessage( {
+					isFailure,
+					reportName: options.reportName,
+					username: options.username,
+					sha: getEnvVar( 'GITHUB_SHA', true ),
+					commitMessage: options.commitMessage,
+					prTitle: options.prTitle,
+					prNumber: options.prNumber,
+					actor: getEnvVar( 'GITHUB_ACTOR', true ),
+					triggeringActor: getEnvVar(
+						'GITHUB_TRIGGERING_ACTOR',
+						true
+					),
+					eventName: getEnvVar( 'GITHUB_EVENT_NAME', true ),
+					runId: getEnvVar( 'GITHUB_RUN_ID', true ),
+					runAttempt: getEnvVar( 'GITHUB_RUN_ATTEMPT', true ),
+					serverUrl: getEnvVar( 'GITHUB_SERVER_URL', true ),
+					repository: getEnvVar( 'GITHUB_REPOSITORY', true ),
+					refType: getEnvVar( 'GITHUB_REF_TYPE', true ),
+					refName: getEnvVar( 'GITHUB_REF_NAME', true ),
+				} );
+
+			Logger.notice( 'Sending new message' );
+			// Send a new main message
+			const response = await postMessage( client, {
+				text: `${ text }`,
+				blocks: mainMsgBlocks,
+				channel,
+				username,
+			} );
+			const mainMessageTS = response.ts;
+
+			if ( detailsMsgBlocksChunks.length > 0 ) {
+				Logger.notice( 'Replying with failure details' );
+				// Send replies to the main message with the current failure result
+				await postMessage( client, {
+					text,
+					blocks: detailsMsgBlocksChunks,
+					channel,
+					username,
+					thread_ts: mainMessageTS,
+				} );
+			}
+		} else {
+			Logger.notice(
+				`No message will be sent for '${ options.conclusion }'`
+			);
+		}
+	} );
+
+export default program;

tools/monorepo-utils/src/slack-test-report/lib/message.ts

@@ -0,0 +1,318 @@
+/**
+ * External dependencies
+ */
+import fs from 'fs';
+import {
+	ChatPostMessageResponse,
+	FilesUploadResponse,
+	WebClient,
+} from '@slack/web-api';
+
+/**
+ * Internal dependencies
+ */
+import { Logger } from '../../core/logger';
+
+interface Options {
+	reportName: string;
+	username: string;
+	isFailure: boolean;
+	eventName: string;
+	sha: string;
+	commitMessage: string;
+	prTitle: string;
+	prNumber: string;
+	actor: string;
+	triggeringActor: string;
+	runId: string;
+	runAttempt: string;
+	serverUrl: string;
+	repository: string;
+	refType: string;
+	refName: string;
+}
+
+/**
+ * Returns a Slack context block element with a given text.
+ *
+ * @param {string} text - the text of the element
+ * @return {Object} - the block element
+ */
+function getTextContextElement( text: string ): object {
+	return {
+		type: 'plain_text',
+		text,
+		emoji: false,
+	};
+}
+
+/**
+ * Returns a Slack button element with a given text and url.
+ *
+ * @param {string} text - the text of the button
+ * @param {string} url  - the url of the button
+ * @return {Object} - the button element
+ */
+function getButton( text: string, url: string ): object {
+	return {
+		type: 'button',
+		text: {
+			type: 'plain_text',
+			text,
+		},
+		url,
+	};
+}
+
+/**
+ * Creates and returns a run url
+ *
+ * @param {Options} options     - the options object
+ * @param {boolean} withAttempt - whether to include the run attempt in the url
+ * @return {string} the run url
+ */
+function getRunUrl( options: Options, withAttempt: boolean ): string {
+	const { serverUrl, runId, repository, runAttempt } = options;
+	return `${ serverUrl }/${ repository }/actions/runs/${ runId }/${
+		withAttempt ? `attempts/${ runAttempt }` : ''
+	}`;
+}
+
+/**
+ * Returns an object with notification data.
+ * Properties: `text` for notification's text and `id` for a unique identifier for the message.
+ * that can be used later on to find this message and update it or send replies.
+ *
+ * @param {Options} options - whether the workflow is failed or not
+ */
+export async function createMessage( options: Options ) {
+	const {
+		sha,
+		eventName,
+		actor,
+		prNumber,
+		prTitle,
+		runId,
+		commitMessage,
+		reportName,
+		runAttempt,
+		triggeringActor,
+		serverUrl,
+		repository,
+		refType,
+		refName,
+	} = options;
+
+	let target = `for ${ sha }`;
+	const contextElements = [];
+	const buttons = [];
+
+	const lastRunBlock = getTextContextElement(
+		`Run: ${ runId }/${ runAttempt }, triggered by ${ triggeringActor }`
+	);
+	const actorBlock = getTextContextElement( `Actor: ${ actor }` );
+	const lastRunButtonBlock = getButton( 'Run', getRunUrl( options, false ) );
+	buttons.push( lastRunButtonBlock );
+
+	if ( eventName === 'pull_request' ) {
+		target = `for pull request *#${ prNumber }*`;
+
+		contextElements.push(
+			getTextContextElement( `Title: ${ prTitle }` ),
+			actorBlock
+		);
+		buttons.push(
+			getButton(
+				`PR #${ prNumber }`,
+				`${ serverUrl }/${ repository }/pull/${ prNumber }`
+			)
+		);
+	}
+
+	if (
+		[ 'push', 'workflow_run', 'workflow_call', 'schedule' ].includes(
+			eventName
+		)
+	) {
+		target = `on ${ refType } _*${ refName }*_ (${ eventName })`;
+		const truncatedMessage =
+			commitMessage.length > 50
+				? commitMessage.substring( 0, 48 ) + '...'
+				: commitMessage;
+
+		contextElements.push(
+			getTextContextElement(
+				`Commit: ${ sha.substring( 0, 8 ) } ${ truncatedMessage }`
+			),
+			actorBlock
+		);
+		buttons.push(
+			getButton(
+				`Commit ${ sha.substring( 0, 8 ) }`,
+				`${ serverUrl }/${ repository }/commit/${ sha }`
+			)
+		);
+	}
+
+	if ( eventName === 'repository_dispatch' ) {
+		target = `for event _*${ eventName }*_`;
+	}
+
+	contextElements.push( lastRunBlock );
+
+	const reportText = reportName ? `_*${ reportName }*_ failed` : 'Failure';
+	const text = `:x:	${ reportText } ${ target }`;
+
+	const mainMsgBlocks = [
+		{
+			type: 'section',
+			text: {
+				type: 'mrkdwn',
+				text,
+			},
+		},
+		{
+			type: 'context',
+			elements: contextElements,
+		},
+		{
+			type: 'actions',
+			elements: buttons,
+		},
+	];
+
+	const detailsMsgBlocksChunks = [];
+	// detailsMsgBlocksChunks.push( ...getPlaywrightBlocks() );
+
+	return { text, mainMsgBlocks, detailsMsgBlocksChunks };
+}
+
+/**
+ * Split an array of blocks into chunks of a given size
+ *
+ * @param {[object]} blocks    - the array to be split
+ * @param {number}   chunkSize - the maximum size of each chunk
+ * @return {any[]} the array of chunks
+ */
+function getBlocksChunksBySize( blocks: any[], chunkSize: number ): any[] {
+	const chunks = [];
+	for ( let i = 0; i < blocks.length; i += chunkSize ) {
+		const chunk = blocks.slice( i, i + chunkSize );
+		chunks.push( chunk );
+	}
+	return chunks;
+}
+
+/**
+ * Split an array of blocks into chunks based on a given type property as delimiter
+ * E.g. if the array is [ {type: 'context'}, {type: 'context'}, {type: 'file'}, {type: 'context'} ] and the delimiter is 'file'
+ * the result will be [ [ {type: 'context'}, {type: 'context'} ], [ {type: 'file'} ], [ {type: 'context'} ] ]
+ *
+ * @param {[object]} blocks - the array to be split
+ * @param {string}   type   - the type property to use as delimiter
+ * @return {any[]} the array of chunks
+ */
+function getBlocksChunksByType( blocks: string | any[], type: string ): any[] {
+	const chunks = [];
+	let nextIndex = 0;
+
+	for ( let i = 0; i < blocks.length; i++ ) {
+		if ( blocks[ i ].type === type ) {
+			if ( nextIndex < i ) {
+				chunks.push( blocks.slice( nextIndex, i ) );
+			}
+			chunks.push( blocks.slice( i, i + 1 ) );
+			nextIndex = i + 1;
+		}
+	}
+
+	if ( nextIndex < blocks.length ) {
+		chunks.push( blocks.slice( nextIndex ) );
+	}
+
+	return chunks;
+}
+
+/**
+ * Split an array of blocks into chunks based on a given type property as delimiter and a max size
+ *
+ * @param {[object]} blocks        - the array to be split
+ * @param {number}   maxSize       - the maximum size of each chunk
+ * @param {string}   typeDelimiter - the type property to use as delimiter
+ * @return {[any]} the array of chunks
+ */
+function getBlocksChunks(
+	blocks: [ object ],
+	maxSize: number,
+	typeDelimiter: string
+): any[] {
+	const chunksByType = getBlocksChunksByType( blocks, typeDelimiter );
+	const chunks = [];
+
+	for ( const chunk of chunksByType ) {
+		// eslint-disable-next-line no-unused-expressions
+		chunk.length > maxSize
+			? chunks.push( ...getBlocksChunksBySize( chunk, maxSize ) )
+			: chunks.push( chunk );
+	}
+
+	return chunks;
+}
+
+export async function postMessage( client: WebClient, options: any ) {
+	const {
+		text,
+		blocks = [],
+		channel,
+		username,
+		icon_emoji,
+		ts,
+		thread_ts,
+	} = options;
+
+	const method = 'postMessage';
+	let response: FilesUploadResponse | ChatPostMessageResponse;
+
+	// Split the blocks into chunks:
+	// - blocks with type 'file' are separate chunks. 'file' type is not a valid block, and when we have one we need to call files.upload instead of chat.postMessage.
+	// - chunk max size is 50 blocks, Slack API will fail if we send more
+	const chunks = getBlocksChunks( blocks, 50, 'file' );
+
+	for ( const chunk of chunks ) {
+		// The expectation is that chunks with files will only have one element
+		if ( chunk[ 0 ].type === 'file' ) {
+			if ( ! fs.existsSync( chunk[ 0 ].path ) ) {
+				Logger.error( 'File not found: ' + chunk[ 0 ].path );
+				continue;
+			}
+
+			try {
+				response = await client.files.upload( {
+					file: fs.createReadStream( chunk[ 0 ].path ),
+					channels: channel,
+					thread_ts,
+				} );
+			} catch ( err ) {
+				Logger.error( err );
+			}
+		} else {
+			try {
+				response = await client.chat[ method ]( {
+					text,
+					blocks: chunk,
+					channel,
+					ts,
+					thread_ts,
+					username,
+					icon_emoji,
+					unfurl_links: false,
+					unfurl_media: false,
+				} );
+			} catch ( err ) {
+				Logger.error( err );
+			}
+		}
+	}
+
+	return response;
+}