From 2fdae2bc0177318576b7a2c857621d5a709ac8f0 Mon Sep 17 00:00:00 2001 From: jamelreid Date: Wed, 8 Dec 2021 12:47:31 -0500 Subject: [PATCH 01/10] Added test-results.json to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 130ad00fe7e..b2d86e53e2a 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ tsconfig.tsbuildinfo /tmp packages/js/e2e-environment/config/default.json packages/js/e2e-environment/docker/wp-cli/initialize.sh + +# Test Results +test-results.json From 5b2480897162df3f509bfb2366bd764d664e99fb Mon Sep 17 00:00:00 2001 From: jamelreid Date: Wed, 8 Dec 2021 12:47:50 -0500 Subject: [PATCH 02/10] Stored test results as json --- packages/js/e2e-environment/bin/e2e-test-integration.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/js/e2e-environment/bin/e2e-test-integration.js b/packages/js/e2e-environment/bin/e2e-test-integration.js index 09f0c95dcf5..e5bd25cfdb9 100755 --- a/packages/js/e2e-environment/bin/e2e-test-integration.js +++ b/packages/js/e2e-environment/bin/e2e-test-integration.js @@ -78,6 +78,8 @@ const jestArgs = [ '--maxWorkers=1', '--rootDir=./', '--verbose', + '--json', + '--outputFile=test-results.json', ...program.args, ]; From 00fc5495ef1610d8ed80c176f1b74009804babf8 Mon Sep 17 00:00:00 2001 From: jamelreid Date: Wed, 8 Dec 2021 12:48:24 -0500 Subject: [PATCH 03/10] Added script to post test results to GitHub PR --- .../bin/post-results-to-github-pr.js | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 packages/js/e2e-environment/bin/post-results-to-github-pr.js diff --git a/packages/js/e2e-environment/bin/post-results-to-github-pr.js b/packages/js/e2e-environment/bin/post-results-to-github-pr.js new file mode 100644 index 00000000000..319c10510dc --- /dev/null +++ b/packages/js/e2e-environment/bin/post-results-to-github-pr.js @@ -0,0 +1,38 @@ +const fs = require( 'fs' ); +const path = require( 'path' ); + +const resultsFile = path.resolve( __dirname, '../test-results.json' ); + +const buildOutput = ( results ) => { + const { TITLE, SMOKE_TEST_URL } = process.env; + + let output = `## ${ TITLE }:\n\n`; + output += `**Test URL:** ${ SMOKE_TEST_URL }\n`; + output += `**Total Number of Passed Tests:** ${ results.numTotalTests }\n`; + output += `**Total Number of Failed Tests:** ${ results.numFailedTests }\n`; + output += `**Total Number of Test Suites:** ${ results.numTotalTestSuites }\n`; + output += `**Total Number of Passed Test Suites:** ${ results.numPassedTestSuites }\n`; + output += `**Total Number of Failed Test Suites:** ${ results.numFailedTestSuites }\n`; + + return output; +}; + +module.exports = async ( { github, context } ) => { + let output = ''; + + if ( fs.existsSync( resultsFile ) ) { + const data = fs.readFileSync( resultsFile ); + const results = JSON.parse( data ); + + output = buildOutput( results ); + } else { + output = `## Test Results Not Found!`; + } + + await github.rest.issues.createComment( { + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output, + } ); +}; From cbf8bfc1d0bff712eb4d56a65ec5ac162a6d57cd Mon Sep 17 00:00:00 2001 From: jamelreid Date: Wed, 8 Dec 2021 12:49:09 -0500 Subject: [PATCH 04/10] Updated workflow to post test results to GitHub PR --- .github/workflows/pr-smoke-test.yml | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index dde107c7df1..bf8dcdb98f1 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -25,6 +25,7 @@ jobs: - name: Install prerequisites. working-directory: package/woocommerce/plugins/woocommerce + id: installation run: | npm install -g pnpm pnpm install @@ -34,6 +35,7 @@ jobs: - name: Run smoke test. working-directory: package/woocommerce/plugins/woocommerce + if: steps.installation.outcome == 'success' env: SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} SMOKE_TEST_ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} @@ -49,8 +51,50 @@ jobs: DEFAULT_TIMEOUT_OVERRIDE: 120000 run: | pnpx wc-e2e test:e2e plugins/woocommerce/tests/e2e/specs/smoke-tests/update-woocommerce.js + + - name: Post Smoke tests results comment on PR + if: always() + uses: actions/github-script@v5 + env: + TITLE: 'External Smoke Test Resutls:' + SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const script = require( './package/woocommerce/packages/js/e2e-environment/bin/post-results-to-github-pr.js' ) + await script({github, context}) + + - name: Run E2E tests. + working-directory: package/woocommerce/plugins/woocommerce + if: steps.installation.outcome == 'success' + env: + SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} + SMOKE_TEST_ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} + SMOKE_TEST_ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} + SMOKE_TEST_ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} + SMOKE_TEST_CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} + SMOKE_TEST_CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} + WC_E2E_SCREENSHOTS: 1 + E2E_RETEST: 1 + E2E_SLACK_TOKEN: ${{ secrets.SMOKE_TEST_SLACK_TOKEN }} + E2E_SLACK_CHANNEL: ${{ secrets.SMOKE_TEST_SLACK_CHANNEL }} + UPDATE_WC: 1 + DEFAULT_TIMEOUT_OVERRIDE: 120000 + run: | pnpx wc-e2e test:e2e + - name: Post E2E tests results comment on PR + if: always() + uses: actions/github-script@v5 + env: + TITLE: 'External E2E Test Resutls:' + SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const script = require( './package/woocommerce/packages/js/e2e-environment/bin/post-results-to-github-pr.js' ) + await script({github, context}) + - name: Remove label from pull request. if: "${{ contains(github.event.pull_request.labels.*.name, 'run: smoke tests') }}" uses: actions-ecosystem/action-remove-labels@v1 From 7b36ba2b3e05f4e22806d888291518d374049a43 Mon Sep 17 00:00:00 2001 From: jamelreid Date: Wed, 8 Dec 2021 13:02:11 -0500 Subject: [PATCH 05/10] Updated CHANGELOG.md --- packages/js/e2e-environment/CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/js/e2e-environment/CHANGELOG.md b/packages/js/e2e-environment/CHANGELOG.md index 781c66330cb..63f199804f0 100644 --- a/packages/js/e2e-environment/CHANGELOG.md +++ b/packages/js/e2e-environment/CHANGELOG.md @@ -2,6 +2,11 @@ ## Added +- Added `post-results-to-github-pr.js` to post smoke test results to a GitHub PR. +- Added jest flags to generate a json test report + +## Added + - Added `await` for every call to `shopper.logout` ## Fixed From a560b4c524030233b60c588e0b00f6825284c498 Mon Sep 17 00:00:00 2001 From: Jamel Noel Reid Date: Tue, 14 Dec 2021 12:50:04 -0500 Subject: [PATCH 06/10] Removed 'External' from Title Co-authored-by: Ron Rennick --- .github/workflows/pr-smoke-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index bf8dcdb98f1..0c3076275d2 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -56,7 +56,7 @@ jobs: if: always() uses: actions/github-script@v5 env: - TITLE: 'External Smoke Test Resutls:' + TITLE: 'Smoke Test Results:' SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} with: github-token: ${{secrets.GITHUB_TOKEN}} From a24a9dcc7d8883839026e1e4725228221823080d Mon Sep 17 00:00:00 2001 From: jamelreid Date: Tue, 14 Dec 2021 12:53:52 -0500 Subject: [PATCH 07/10] Removed 'External' from step title --- .github/workflows/pr-smoke-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index 0c3076275d2..59a146a3db8 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -87,7 +87,7 @@ jobs: if: always() uses: actions/github-script@v5 env: - TITLE: 'External E2E Test Resutls:' + TITLE: 'E2E Test Results:' SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} with: github-token: ${{secrets.GITHUB_TOKEN}} From e5c432e58931d6e8a865d952a47467cd678b8ac0 Mon Sep 17 00:00:00 2001 From: jamelreid Date: Wed, 15 Dec 2021 17:30:52 -0500 Subject: [PATCH 08/10] Updated TITLE env variable --- .github/workflows/pr-smoke-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index 59a146a3db8..72f6ad376fd 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -56,7 +56,7 @@ jobs: if: always() uses: actions/github-script@v5 env: - TITLE: 'Smoke Test Results:' + TITLE: 'Smoke Test Results' SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -87,7 +87,7 @@ jobs: if: always() uses: actions/github-script@v5 env: - TITLE: 'E2E Test Results:' + TITLE: 'E2E Test Results' SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} with: github-token: ${{secrets.GITHUB_TOKEN}} From 69f6850af095c01029b62b024e5b48b2e5d0bc38 Mon Sep 17 00:00:00 2001 From: jamelreid Date: Wed, 15 Dec 2021 17:48:33 -0500 Subject: [PATCH 09/10] Updated step condition --- .github/workflows/pr-smoke-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index 72f6ad376fd..a1886be760b 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -53,7 +53,7 @@ jobs: pnpx wc-e2e test:e2e plugins/woocommerce/tests/e2e/specs/smoke-tests/update-woocommerce.js - name: Post Smoke tests results comment on PR - if: always() + if: steps.installation.outcome == 'success' uses: actions/github-script@v5 env: TITLE: 'Smoke Test Results' @@ -84,7 +84,7 @@ jobs: pnpx wc-e2e test:e2e - name: Post E2E tests results comment on PR - if: always() + if: steps.installation.outcome == 'success' uses: actions/github-script@v5 env: TITLE: 'E2E Test Results' From 237bb5ecd5d5f30349832c7bbeb88449a22b2133 Mon Sep 17 00:00:00 2001 From: jamelreid Date: Mon, 20 Dec 2021 13:10:20 -0500 Subject: [PATCH 10/10] Updated step condition --- .github/workflows/pr-smoke-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index a1886be760b..72f6ad376fd 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -53,7 +53,7 @@ jobs: pnpx wc-e2e test:e2e plugins/woocommerce/tests/e2e/specs/smoke-tests/update-woocommerce.js - name: Post Smoke tests results comment on PR - if: steps.installation.outcome == 'success' + if: always() uses: actions/github-script@v5 env: TITLE: 'Smoke Test Results' @@ -84,7 +84,7 @@ jobs: pnpx wc-e2e test:e2e - name: Post E2E tests results comment on PR - if: steps.installation.outcome == 'success' + if: always() uses: actions/github-script@v5 env: TITLE: 'E2E Test Results'