Merge pull request #31397 from woocommerce/add/post-external-smoke-test-results-to-pr

Post external smoke test results to GitHub PR

.github/workflows/pr-smoke-test.yml

@@ -25,6 +25,7 @@ jobs:
 
       - name: Install prerequisites.
         working-directory: package/woocommerce/plugins/woocommerce
+        id: installation
         run: |
           npm install -g pnpm
           pnpm install
@@ -34,6 +35,7 @@ jobs:
           
       - name: Run smoke test.
         working-directory: package/woocommerce/plugins/woocommerce
+        if: steps.installation.outcome == 'success'
         env:
           SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }}
           SMOKE_TEST_ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }}
@@ -49,8 +51,50 @@ jobs:
           DEFAULT_TIMEOUT_OVERRIDE: 120000
         run: |
           pnpx wc-e2e test:e2e plugins/woocommerce/tests/e2e/specs/smoke-tests/update-woocommerce.js
+
+      - name: Post Smoke tests results comment on PR
+        if: always()
+        uses: actions/github-script@v5
+        env: 
+          TITLE: 'Smoke Test Results'
+          SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }}
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const script = require( './package/woocommerce/packages/js/e2e-environment/bin/post-results-to-github-pr.js' )
+            await script({github, context})
+
+      - name: Run E2E tests.
+        working-directory: package/woocommerce/plugins/woocommerce
+        if: steps.installation.outcome == 'success'
+        env:
+          SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }}
+          SMOKE_TEST_ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }}
+          SMOKE_TEST_ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }}
+          SMOKE_TEST_ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }}
+          SMOKE_TEST_CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }}
+          SMOKE_TEST_CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }}
+          WC_E2E_SCREENSHOTS: 1
+          E2E_RETEST: 1
+          E2E_SLACK_TOKEN: ${{ secrets.SMOKE_TEST_SLACK_TOKEN }}
+          E2E_SLACK_CHANNEL: ${{ secrets.SMOKE_TEST_SLACK_CHANNEL }}
+          UPDATE_WC: 1
+          DEFAULT_TIMEOUT_OVERRIDE: 120000
+        run: |
           pnpx wc-e2e test:e2e
 
+      - name: Post E2E tests results comment on PR
+        if: always()
+        uses: actions/github-script@v5
+        env: 
+          TITLE: 'E2E Test Results'
+          SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }}
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const script = require( './package/woocommerce/packages/js/e2e-environment/bin/post-results-to-github-pr.js' )
+            await script({github, context})
+
       - name: Remove label from pull request.
         if: "${{ contains(github.event.pull_request.labels.*.name, 'run: smoke tests') }}"
         uses: actions-ecosystem/action-remove-labels@v1

.gitignore

@@ -70,3 +70,6 @@ yarn.lock
 
 # Editors
 nbproject/private/
+
+# Test Results
+test-results.json

packages/js/e2e-environment/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Added
 
+- Added `post-results-to-github-pr.js` to post smoke test results to a GitHub PR.
+- Added jest flags to generate a json test report
+
+## Added
+
 - Added `await` for every call to `shopper.logout`
 - Updated `getLatestReleaseZipUrl()` to allow passing in an authorization token and simplified arguments to just the repository name
 - Added `upload.ini` which increases the limits for uploading files (such as for plugins) in the Docker environment

packages/js/e2e-environment/bin/e2e-test-integration.js

@@ -79,6 +79,8 @@ const jestArgs = [
 	'--maxWorkers=1',
 	'--rootDir=./',
 	'--verbose',
+	'--json',
+	'--outputFile=test-results.json',
 	...program.args,
 ];
 

packages/js/e2e-environment/bin/post-results-to-github-pr.js

@@ -0,0 +1,38 @@
+const fs = require( 'fs' );
+const path = require( 'path' );
+
+const resultsFile = path.resolve( __dirname, '../test-results.json' );
+
+const buildOutput = ( results ) => {
+	const { TITLE, SMOKE_TEST_URL } = process.env;
+
+	let output = `## ${ TITLE }:\n\n`;
+	output += `**Test URL:** ${ SMOKE_TEST_URL }\n`;
+	output += `**Total Number of Passed Tests:** ${ results.numTotalTests }\n`;
+	output += `**Total Number of Failed Tests:** ${ results.numFailedTests }\n`;
+	output += `**Total Number of Test Suites:** ${ results.numTotalTestSuites }\n`;
+	output += `**Total Number of Passed Test Suites:** ${ results.numPassedTestSuites }\n`;
+	output += `**Total Number of Failed Test Suites:** ${ results.numFailedTestSuites }\n`;
+
+	return output;
+};
+
+module.exports = async ( { github, context } ) => {
+	let output = '';
+
+	if ( fs.existsSync( resultsFile ) ) {
+		const data = fs.readFileSync( resultsFile );
+		const results = JSON.parse( data );
+
+		output = buildOutput( results );
+	} else {
+		output = `## Test Results Not Found!`;
+	}
+
+	await github.rest.issues.createComment( {
+		issue_number: context.issue.number,
+		owner: context.repo.owner,
+		repo: context.repo.repo,
+		body: output,
+	} );
+};