Steve-Dee-Designs
/
woocommerce
mirror of https://github.com/woocommerce/woocommerce.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Consistent folder structure for E2E and API test results (#35907) 

Consistent folder structure for E2E and REST API test results
This commit is contained in:
rodelgc 2022-12-19 14:33:22 +08:00 committed by GitHub
parent 76810a4445
commit c4260c935f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 148 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/pr-build-and-e2e-tests.yml vendored
View File

@ -12,8 +12,8 @@ jobs:
name: Runs E2E tests. name: Runs E2E tests.
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
env: env:
ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
outputs: outputs:
E2E_GRAND_TOTAL: ${{ steps.count_e2e_total.outputs.E2E_GRAND_TOTAL }} E2E_GRAND_TOTAL: ${{ steps.count_e2e_total.outputs.E2E_GRAND_TOTAL }}
steps: steps:
@ -80,8 +80,8 @@ jobs:
name: Runs API tests. name: Runs API tests.
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
env: env:
ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results
ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3

4
.github/workflows/smoke-test-daily.yml vendored
View File

@ -79,8 +79,8 @@ jobs:
needs: [e2e-tests] needs: [e2e-tests]
if: success() || failure() if: success() || failure()
env: env:
ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results
ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:

11
.gitignore vendored
View File

@ -72,8 +72,8 @@ yarn.lock
# Editors # Editors
nbproject/private/ nbproject/private/
# Test Results # E2E and API Test Results
test-results.json test-results
# Admin Feature config # Admin Feature config
plugins/woocommerce/includes/react-admin/feature-config.php plugins/woocommerce/includes/react-admin/feature-config.php
@ -89,13 +89,6 @@ allure-results
changes.json changes.json
.env .env
# Playwright output & working files
/plugins/woocommerce/tests/e2e-pw/output
/plugins/woocommerce/tests/e2e-pw/report
/plugins/woocommerce/tests/e2e-pw/storage
/plugins/woocommerce/tests/e2e-pw/test-results.json
/plugins/woocommerce/tests/api-core-tests/output
# Turborepo # Turborepo
.turbo .turbo

4
plugins/woocommerce/changelog/e2e-output-directories Normal file
View File

@ -0,0 +1,4 @@
Significance: patch
Type: dev
Consistent folder structure for E2E and API test results

61
plugins/woocommerce/tests/api-core-tests/README.md
View File

@ -12,6 +12,9 @@ This package contains automated API tests for WooCommerce, based on Playwright a
- [Tools for writing tests](#tools-for-writing-tests) - [Tools for writing tests](#tools-for-writing-tests)
- [Creating test structure](#creating-test-structure) - [Creating test structure](#creating-test-structure)
- [Writing the test](#writing-the-test) - [Writing the test](#writing-the-test)
- [Guide for using test reports](#guide-for-using-test-reports)
- [Viewing the Playwright HTML report](#viewing-the-playwright-html-report)
- [Viewing the Allure report](#viewing-the-allure-report)
- [Debugging tests](#debugging-tests) - [Debugging tests](#debugging-tests)
## Pre-requisites ## Pre-requisites
@ -146,6 +149,64 @@ test.describe( 'Merchant can create virtual product', () => {
} ); } );
``` ```
## Guide for using test reports
The tests would generate three kinds of reports after the run:
1. A Playwright HTML report.
1. A Playwright JSON report.
1. Allure results.
By default, they are saved inside the `test-results` folder. If you want to save them in a custom location, just assign the absolute path to the environment variables mentioned in the [Playwright](https://playwright.dev/docs/test-reporters) and [Allure-Playwright](https://www.npmjs.com/package/allure-playwright) documentation.
| Report | Default location | Environment variable for custom location |
| ----------- | ---------------- | ---------------------------------------- |
| Playwright HTML report | `test-results/playwright-report` | `PLAYWRIGHT_HTML_REPORT` |
| Playwright JSON report | `test-results/test-results.json` | `PLAYWRIGHT_JSON_OUTPUT_NAME` |
| Allure results | `test-results/allure-results` | `ALLURE_RESULTS_DIR` |
### Viewing the Playwright HTML report
Use the `playwright show-report $PATH_TO_PLAYWRIGHT_HTML_REPORT` command to open the report. For example, assuming that you're at the root of the WooCommerce monorepo, and that you did not specify a custom location for the report, you would use the following commands:
```bash
cd plugins/woocommerce
pnpm exec playwright show-report tests/api-core-tests/test-results/playwright-report
```
For more details about the Playwright HTML report, see their [HTML Reporter](https://playwright.dev/docs/test-reporters#html-reporter) documentation.
### Viewing the Allure report
This assumes that you're already familiar with reports generated by the [Allure Framework](https://github.com/allure-framework), particularly:
- What the `allure-results` and `allure-report` folders are, and how they're different from each other.
- Allure commands like `allure generate` and `allure open`.
Use the `allure generate` command to generate an HTML report from the `allure-results` directory created at the end of the test run. Then, use the `allure open` command to open it on your browser. For example, assuming that:
- You're at the root of the WooCommerce monorepo
- You did not specify a custom location for `allure-results` (you did not assign a value to `ALLURE_RESULTS_DIR`)
- You want to generate the `allure-report` folder in `plugins/woocommerce/tests/api-core-tests/test-results`
Then you would need to use the following commands:
```bash
cd plugins/woocommerce
pnpm exec allure generate --clean tests/api-core-tests/test-results/allure-results --output tests/api-core-tests/test-results/allure-report
pnpm exec allure open tests/api-core-tests/test-results/allure-report
```
A browser window should open the Allure report.
If you're using [WSL](https://learn.microsoft.com/en-us/windows/wsl/about) however, you might get this message right after running the `allure open` command:
```
Starting web server...
2022-12-09 18:52:01.323:INFO::main: Logging initialized @286ms to org.eclipse.jetty.util.log.StdErrLog
Can not open browser because this capability is not supported on your platform. You can use the link below to open the report manually.
Server started at <http://127.0.1.1:38917/>. Press <Ctrl+C> to exit
```
In this case, take note of the port number (38917 in the example above) and then use it to navigate to `http://localhost`. Taking the example above, you should be able to view the Allure report on http://localhost:38917.
To know more about the allure-playwright integration, see their [GitHub documentation](https://github.com/allure-framework/allure-js/tree/master/packages/allure-playwright).
## Debugging tests ## Debugging tests
For Playwright debugging, follow [Playwright's documentation](https://playwright.dev/docs/debug). For Playwright debugging, follow [Playwright's documentation](https://playwright.dev/docs/debug).

10
plugins/woocommerce/tests/api-core-tests/playwright.config.js
View File

@ -19,7 +19,7 @@ const config = {
? Number( DEFAULT_TIMEOUT_OVERRIDE ) ? Number( DEFAULT_TIMEOUT_OVERRIDE )
: 90 * 1000, : 90 * 1000,
expect: { timeout: 20 * 1000 }, expect: { timeout: 20 * 1000 },
outputDir: './report', outputDir: './test-results/report',
testDir: 'tests', testDir: 'tests',
retries: CI ? 4 : 2, retries: CI ? 4 : 2,
workers: 4, workers: 4,
@ -28,7 +28,7 @@ const config = {
[ [
'html', 'html',
{ {
outputFolder: 'output', outputFolder: process.env.PLAYWRIGHT_HTML_REPORT ?? './test-results/playwright-report',
open: CI ? 'never' : 'always', open: CI ? 'never' : 'always',
}, },
], ],
@ -37,14 +37,14 @@ const config = {
{ {
outputFolder: outputFolder:
process.env.ALLURE_RESULTS_DIR ?? process.env.ALLURE_RESULTS_DIR ??
'tests/api-core-tests/api-test-report/allure-results', './tests/api-core-tests/test-results/allure-results',
}, },
], ],
[ [
'json', 'json',
{ {
outputFile: outputFile: process.env.PLAYWRIGHT_JSON_OUTPUT_NAME ??
'tests/api-core-tests/api-test-report/test-results.json', './test-results/test-results.json',
}, },
], ],
], ],

61
plugins/woocommerce/tests/e2e-pw/README.md
View File

@ -12,6 +12,9 @@ This is the documentation for the new e2e testing setup based on Playwright and
- [Tools for writing tests](#tools-for-writing-tests) - [Tools for writing tests](#tools-for-writing-tests)
- [Creating test structure](#creating-test-structure) - [Creating test structure](#creating-test-structure)
- [Writing the test](#writing-the-test) - [Writing the test](#writing-the-test)
- [Guide for using test reports](#guide-for-using-test-reports)
- [Viewing the Playwright HTML report](#viewing-the-playwright-html-report)
- [Viewing the Allure report](#viewing-the-allure-report)
- [Debugging tests](#debugging-tests) - [Debugging tests](#debugging-tests)
## Pre-requisites ## Pre-requisites
@ -140,6 +143,64 @@ test.describe( 'Merchant can create virtual product', () => {
} ); } );
``` ```
## Guide for using test reports
The tests would generate three kinds of reports after the run:
1. A Playwright HTML report.
1. A Playwright JSON report.
1. Allure results.
By default, they are saved inside the `test-results` folder. If you want to save them in a custom location, just assign the absolute path to the environment variables mentioned in the [Playwright](https://playwright.dev/docs/test-reporters) and [Allure-Playwright](https://www.npmjs.com/package/allure-playwright) documentation.
| Report | Default location | Environment variable for custom location |
| ----------- | ---------------- | ---------------------------------------- |
| Playwright HTML report | `test-results/playwright-report` | `PLAYWRIGHT_HTML_REPORT` |
| Playwright JSON report | `test-results/test-results.json` | `PLAYWRIGHT_JSON_OUTPUT_NAME` |
| Allure results | `test-results/allure-results` | `ALLURE_RESULTS_DIR` |
### Viewing the Playwright HTML report
Use the `playwright show-report $PATH_TO_PLAYWRIGHT_HTML_REPORT` command to open the report. For example, assuming that you're at the root of the WooCommerce monorepo, and that you did not specify a custom location for the report, you would use the following commands:
```bash
cd plugins/woocommerce
pnpm exec playwright show-report tests/e2e-pw/test-results/playwright-report
```
For more details about the Playwright HTML report, see their [HTML Reporter](https://playwright.dev/docs/test-reporters#html-reporter) documentation.
### Viewing the Allure report
This assumes that you're already familiar with reports generated by the [Allure Framework](https://github.com/allure-framework), particularly:
- What the `allure-results` and `allure-report` folders are, and how they're different from each other.
- Allure commands like `allure generate` and `allure open`.
Use the `allure generate` command to generate an HTML report from the `allure-results` directory created at the end of the test run. Then, use the `allure open` command to open it on your browser. For example, assuming that:
- You're at the root of the WooCommerce monorepo
- You did not specify a custom location for `allure-results` (you did not assign a value to `ALLURE_RESULTS_DIR`)
- You want to generate the `allure-report` folder in `plugins/woocommerce/tests/e2e-pw/test-results`
Then you would need to use the following commands:
```bash
cd plugins/woocommerce
pnpm exec allure generate --clean tests/e2e-pw/test-results/allure-results --output tests/e2e-pw/test-results/allure-report
pnpm exec allure open tests/e2e-pw/test-results/allure-report
```
A browser window should open the Allure report.
If you're on [WSL](https://learn.microsoft.com/en-us/windows/wsl/about) however, you might get this message right after running the `allure open` command:
```
Starting web server...
2022-12-09 18:52:01.323:INFO::main: Logging initialized @286ms to org.eclipse.jetty.util.log.StdErrLog
Can not open browser because this capability is not supported on your platform. You can use the link below to open the report manually.
Server started at <http://127.0.1.1:38917/>. Press <Ctrl+C> to exit
```
In this case, take note of the port number (38917 in the example above) and then use it to navigate to `http://localhost`. Taking the example above, you should be able to view the Allure report on http://localhost:38917.
To know more about the allure-playwright integration, see their [GitHub documentation](https://github.com/allure-framework/allure-js/tree/master/packages/allure-playwright).
## Debugging tests ## Debugging tests
For Playwright debugging, follow [Playwright's documentation](https://playwright.dev/docs/debug). For Playwright debugging, follow [Playwright's documentation](https://playwright.dev/docs/debug).

14
plugins/woocommerce/tests/e2e-pw/playwright.config.js
View File

@ -5,6 +5,7 @@ const {
CI, CI,
DEFAULT_TIMEOUT_OVERRIDE, DEFAULT_TIMEOUT_OVERRIDE,
E2E_MAX_FAILURES, E2E_MAX_FAILURES,
PLAYWRIGHT_HTML_REPORT,
} = process.env; } = process.env;
const config = { const config = {
@ -12,7 +13,7 @@ const config = {
? Number( DEFAULT_TIMEOUT_OVERRIDE ) ? Number( DEFAULT_TIMEOUT_OVERRIDE )
: 90 * 1000, : 90 * 1000,
expect: { timeout: 20 * 1000 }, expect: { timeout: 20 * 1000 },
outputDir: './report', outputDir: './test-results/report',
globalSetup: require.resolve( './global-setup' ), globalSetup: require.resolve( './global-setup' ),
globalTeardown: require.resolve( './global-teardown' ), globalTeardown: require.resolve( './global-teardown' ),
testDir: 'tests', testDir: 'tests',
@ -23,7 +24,9 @@ const config = {
[ [
'html', 'html',
{ {
outputFolder: 'output', outputFolder:
PLAYWRIGHT_HTML_REPORT ??
'./test-results/playwright-report',
open: CI ? 'never' : 'always', open: CI ? 'never' : 'always',
}, },
], ],
@ -31,18 +34,19 @@ const config = {
'allure-playwright', 'allure-playwright',
{ {
outputFolder: outputFolder:
ALLURE_RESULTS_DIR ?? 'tests/e2e-pw/allure-results', ALLURE_RESULTS_DIR ??
'./tests/e2e-pw/test-results/allure-results',
detail: true, detail: true,
suiteTitle: true, suiteTitle: true,
}, },
], ],
[ 'json', { outputFile: 'tests/e2e-pw/test-results.json' } ], [ 'json', { outputFile: './test-results/test-results.json' } ],
], ],
maxFailures: E2E_MAX_FAILURES ? Number( E2E_MAX_FAILURES ) : 0, maxFailures: E2E_MAX_FAILURES ? Number( E2E_MAX_FAILURES ) : 0,
use: { use: {
baseURL: BASE_URL ?? 'http://localhost:8086', baseURL: BASE_URL ?? 'http://localhost:8086',
screenshot: 'only-on-failure', screenshot: 'only-on-failure',
stateDir: 'tests/e2e-pw/storage/', stateDir: 'tests/e2e-pw/test-results/storage/',
trace: 'retain-on-failure', trace: 'retain-on-failure',
video: 'on-first-retry', video: 'on-first-retry',
viewport: { width: 1280, height: 720 }, viewport: { width: 1280, height: 720 },