diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index dde107c7df1..72f6ad376fd 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -25,6 +25,7 @@ jobs: - name: Install prerequisites. working-directory: package/woocommerce/plugins/woocommerce + id: installation run: | npm install -g pnpm pnpm install @@ -34,6 +35,7 @@ jobs: - name: Run smoke test. working-directory: package/woocommerce/plugins/woocommerce + if: steps.installation.outcome == 'success' env: SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} SMOKE_TEST_ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} @@ -49,8 +51,50 @@ jobs: DEFAULT_TIMEOUT_OVERRIDE: 120000 run: | pnpx wc-e2e test:e2e plugins/woocommerce/tests/e2e/specs/smoke-tests/update-woocommerce.js + + - name: Post Smoke tests results comment on PR + if: always() + uses: actions/github-script@v5 + env: + TITLE: 'Smoke Test Results' + SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const script = require( './package/woocommerce/packages/js/e2e-environment/bin/post-results-to-github-pr.js' ) + await script({github, context}) + + - name: Run E2E tests. + working-directory: package/woocommerce/plugins/woocommerce + if: steps.installation.outcome == 'success' + env: + SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} + SMOKE_TEST_ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} + SMOKE_TEST_ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} + SMOKE_TEST_ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} + SMOKE_TEST_CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} + SMOKE_TEST_CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} + WC_E2E_SCREENSHOTS: 1 + E2E_RETEST: 1 + E2E_SLACK_TOKEN: ${{ secrets.SMOKE_TEST_SLACK_TOKEN }} + E2E_SLACK_CHANNEL: ${{ secrets.SMOKE_TEST_SLACK_CHANNEL }} + UPDATE_WC: 1 + DEFAULT_TIMEOUT_OVERRIDE: 120000 + run: | pnpx wc-e2e test:e2e + - name: Post E2E tests results comment on PR + if: always() + uses: actions/github-script@v5 + env: + TITLE: 'E2E Test Results' + SMOKE_TEST_URL: ${{ secrets.SMOKE_TEST_URL }} + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const script = require( './package/woocommerce/packages/js/e2e-environment/bin/post-results-to-github-pr.js' ) + await script({github, context}) + - name: Remove label from pull request. if: "${{ contains(github.event.pull_request.labels.*.name, 'run: smoke tests') }}" uses: actions-ecosystem/action-remove-labels@v1 diff --git a/.gitignore b/.gitignore index 257b401098d..f6fa98bdbca 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,6 @@ yarn.lock # Editors nbproject/private/ + +# Test Results +test-results.json diff --git a/packages/js/e2e-environment/CHANGELOG.md b/packages/js/e2e-environment/CHANGELOG.md index fa501395906..f3aabfc7a84 100644 --- a/packages/js/e2e-environment/CHANGELOG.md +++ b/packages/js/e2e-environment/CHANGELOG.md @@ -2,6 +2,11 @@ ## Added +- Added `post-results-to-github-pr.js` to post smoke test results to a GitHub PR. +- Added jest flags to generate a json test report + +## Added + - Added `await` for every call to `shopper.logout` - Updated `getLatestReleaseZipUrl()` to allow passing in an authorization token and simplified arguments to just the repository name - Added `upload.ini` which increases the limits for uploading files (such as for plugins) in the Docker environment diff --git a/packages/js/e2e-environment/bin/e2e-test-integration.js b/packages/js/e2e-environment/bin/e2e-test-integration.js index be837564888..caf1c85671b 100755 --- a/packages/js/e2e-environment/bin/e2e-test-integration.js +++ b/packages/js/e2e-environment/bin/e2e-test-integration.js @@ -79,6 +79,8 @@ const jestArgs = [ '--maxWorkers=1', '--rootDir=./', '--verbose', + '--json', + '--outputFile=test-results.json', ...program.args, ]; diff --git a/packages/js/e2e-environment/bin/post-results-to-github-pr.js b/packages/js/e2e-environment/bin/post-results-to-github-pr.js new file mode 100644 index 00000000000..319c10510dc --- /dev/null +++ b/packages/js/e2e-environment/bin/post-results-to-github-pr.js @@ -0,0 +1,38 @@ +const fs = require( 'fs' ); +const path = require( 'path' ); + +const resultsFile = path.resolve( __dirname, '../test-results.json' ); + +const buildOutput = ( results ) => { + const { TITLE, SMOKE_TEST_URL } = process.env; + + let output = `## ${ TITLE }:\n\n`; + output += `**Test URL:** ${ SMOKE_TEST_URL }\n`; + output += `**Total Number of Passed Tests:** ${ results.numTotalTests }\n`; + output += `**Total Number of Failed Tests:** ${ results.numFailedTests }\n`; + output += `**Total Number of Test Suites:** ${ results.numTotalTestSuites }\n`; + output += `**Total Number of Passed Test Suites:** ${ results.numPassedTestSuites }\n`; + output += `**Total Number of Failed Test Suites:** ${ results.numFailedTestSuites }\n`; + + return output; +}; + +module.exports = async ( { github, context } ) => { + let output = ''; + + if ( fs.existsSync( resultsFile ) ) { + const data = fs.readFileSync( resultsFile ); + const results = JSON.parse( data ); + + output = buildOutput( results ); + } else { + output = `## Test Results Not Found!`; + } + + await github.rest.issues.createComment( { + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output, + } ); +};