diff --git a/packages/js/e2e-environment/README.md b/packages/js/e2e-environment/README.md index 26752959fba..85a2e0a76a9 100644 --- a/packages/js/e2e-environment/README.md +++ b/packages/js/e2e-environment/README.md @@ -71,6 +71,20 @@ module.exports = jestConfig; **NOTE:** Your project's Jest config file is: `tests/e2e/config/jest.config.js`. +### The Jest Object + +The E2E environment has the following methods to let us control Jest's overall behavior. + +| Function | Parameters | Description | +|-----------|-------------|--------------| +| `setupJestRetries` | `retries` | Sets the amount of retries on failed tests + +**NOTE:** The amount of times failed tests are retried can also be set using the `E2E_RETRY_TIMES` environment variable when executing tests. This can be done using the command below: + +``` +E2E_RETRY_TIMES=2 pnpx wc-e2e test:e2e +``` + #### Test Screenshots The test sequencer provides a screenshot function for test failures. To enable screenshots on test failure use diff --git a/packages/js/e2e-environment/config/index.js b/packages/js/e2e-environment/config/index.js index b53bdd83640..ba96035504a 100644 --- a/packages/js/e2e-environment/config/index.js +++ b/packages/js/e2e-environment/config/index.js @@ -3,15 +3,17 @@ */ const jestConfig = require( './jest.config' ); const jestPuppeteerConfig = require( './jest-puppeteer.config' ); +const jestobjectConfig = require('./jest-object.config'); const { useE2EBabelConfig, useE2EEsLintConfig, useE2EJestConfig, - useE2EJestPuppeteerConfig + useE2EJestPuppeteerConfig, } = require( './use-config' ); module.exports = { jestConfig, + ...jestobjectConfig, jestPuppeteerConfig, useE2EBabelConfig, useE2EEsLintConfig, diff --git a/packages/js/e2e-environment/config/jest-object.config.js b/packages/js/e2e-environment/config/jest-object.config.js new file mode 100644 index 00000000000..e165e42edeb --- /dev/null +++ b/packages/js/e2e-environment/config/jest-object.config.js @@ -0,0 +1,20 @@ +/** + * External Dependencies + */ +const { E2E_RETRY_TIMES } = process.env; + +const setupJestRetries = ( retries = 2 ) => { + const retryTimes = E2E_RETRY_TIMES ? E2E_RETRY_TIMES : retries; + + jest.retryTimes( retryTimes ); +}; + +// If more methods are added to setupJestObject, it should be include in the readme +const setupJestObject = () => { + setupJestRetries(); +}; + +module.exports = { + setupJestObject, + setupJestRetries, +}; diff --git a/packages/js/e2e-environment/config/jest.config.js b/packages/js/e2e-environment/config/jest.config.js index c0505439b60..05fda6e707d 100644 --- a/packages/js/e2e-environment/config/jest.config.js +++ b/packages/js/e2e-environment/config/jest.config.js @@ -55,6 +55,9 @@ const combinedConfig = { transformIgnorePatterns: [ 'node_modules/(?!(woocommerce)/)', ], + + testRunner: 'jest-circus/runner', + roots: [ testSpecs ], }; diff --git a/packages/js/e2e-environment/package.json b/packages/js/e2e-environment/package.json index 33e576f5420..8955bc1409c 100644 --- a/packages/js/e2e-environment/package.json +++ b/packages/js/e2e-environment/package.json @@ -32,8 +32,8 @@ "jest": "^25.1.0", "jest-each": "25.5.0", "jest-puppeteer": "^4.4.0", - "request": "^2.88.2", - "node-stream-zip": "^1.13.6" + "node-stream-zip": "^1.13.6", + "request": "^2.88.2" }, "devDependencies": { "@babel/cli": "7.12.8", @@ -42,6 +42,7 @@ "@babel/preset-env": "7.12.7", "@wordpress/eslint-plugin": "7.3.0", "eslint": "^8.1.0", + "jest-circus": "25.1.0", "ndb": "^1.1.5", "semver": "^7.3.2" }, diff --git a/packages/js/e2e-environment/src/setup/jest.setup.js b/packages/js/e2e-environment/src/setup/jest.setup.js index 2481a9b5771..f7ed5466984 100644 --- a/packages/js/e2e-environment/src/setup/jest.setup.js +++ b/packages/js/e2e-environment/src/setup/jest.setup.js @@ -9,6 +9,7 @@ import { setBrowserViewport, } from '@wordpress/e2e-test-utils'; import { consoleShouldSuppress, addConsoleSuppression } from '../../utils'; +import { setupJestRetries } from '../../config/jest-object.config'; /** * Array of page event tuples of [ eventName, handler ]. @@ -175,6 +176,7 @@ beforeAll( async () => { capturePageEventsForTearDown(); enablePageDialogAccept(); observeConsoleLogging(); + setupJestRetries(); } ); afterEach( async () => { diff --git a/packages/js/e2e-utils/CHANGELOG.md b/packages/js/e2e-utils/CHANGELOG.md index c40bb45e486..00d16887d64 100644 --- a/packages/js/e2e-utils/CHANGELOG.md +++ b/packages/js/e2e-utils/CHANGELOG.md @@ -1,5 +1,9 @@ # Unreleased +## Fixed + +- Identified the default product category using `slug == 'uncategorized'` in `deleteAllProductCategories` + ## Changes - Removed `page.waitForNavigation()` from `shopper.logout()` diff --git a/packages/js/e2e-utils/src/flows/with-rest-api.js b/packages/js/e2e-utils/src/flows/with-rest-api.js index 5fc127ee0e5..ce15c98ce29 100644 --- a/packages/js/e2e-utils/src/flows/with-rest-api.js +++ b/packages/js/e2e-utils/src/flows/with-rest-api.js @@ -138,7 +138,7 @@ export const withRestApi = { if ( productCategories.data && productCategories.data.length ) { for ( let c = 0; c < productCategories.data.length; c++ ) { // The default `uncategorized` category can't be deleted - if ( productCategories.data[c].id == 0 ) { + if ( productCategories.data[c].slug == 'uncategorized' ) { continue; } const response = await client.delete( productCategoriesPath + `/${productCategories.data[c].id}?force=true` ); diff --git a/plugins/woocommerce/tests/e2e/README.md b/plugins/woocommerce/tests/e2e/README.md index dfd265a4010..0f734809c01 100644 --- a/plugins/woocommerce/tests/e2e/README.md +++ b/plugins/woocommerce/tests/e2e/README.md @@ -186,6 +186,15 @@ For example: - `PUPPETEER_SLOWMO=10` - will run tests faster - `PUPPETEER_SLOWMO=70` - will run tests slower +### How to retry failed tests + +Sometimes tests may fail for different reasons such as network issues, or lost connection. To mitigate against test flakiess, failed tests are rerun up to 3 times before being marked as failed. The amount of retry attempts can be adjusted by passing the `E2E_RETRY_TIMES` variable when running tests. For example: + +``` +cd plugins/woocommerce +E2E_RETRY_TIMES=2 pnpx wc-e2e test:e2e +``` + ### How to run tests in debug mode Tests run in headless mode by default. While writing tests it may be useful to have the debugger loaded while running a test in non-headless mode. To run tests in debug mode: diff --git a/plugins/woocommerce/tests/e2e/config/jest.setup.js b/plugins/woocommerce/tests/e2e/config/jest.setup.js index 5fd474f804f..1d2ba572ee4 100644 --- a/plugins/woocommerce/tests/e2e/config/jest.setup.js +++ b/plugins/woocommerce/tests/e2e/config/jest.setup.js @@ -7,7 +7,7 @@ import { const config = require( 'config' ); const { HTTPClientFactory } = require( '@woocommerce/api' ); -const { addConsoleSuppression, updateReadyPageStatus } = require( '@woocommerce/e2e-environment' ); +const { addConsoleSuppression, updateReadyPageStatus, setupJestRetries } = require( '@woocommerce/e2e-environment' ); const { DEFAULT_TIMEOUT_OVERRIDE } = process.env; // @todo: remove this once https://github.com/woocommerce/woocommerce-admin/issues/6992 has been addressed @@ -40,6 +40,8 @@ async function trashExistingPosts() { // each other's side-effects. beforeAll(async () => { + setupJestRetries(); + if ( DEFAULT_TIMEOUT_OVERRIDE ) { page.setDefaultNavigationTimeout( DEFAULT_TIMEOUT_OVERRIDE ); page.setDefaultTimeout( DEFAULT_TIMEOUT_OVERRIDE ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fc6a9fd1b00..183937402f8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -125,6 +125,7 @@ importers: commander: 4.1.1 eslint: ^8.1.0 jest: ^25.1.0 + jest-circus: 25.1.0 jest-each: 25.5.0 jest-puppeteer: ^4.4.0 ndb: ^1.1.5 @@ -152,6 +153,7 @@ importers: '@babel/preset-env': 7.12.7_@babel+core@7.12.9 '@wordpress/eslint-plugin': 7.3.0_eslint@8.1.0+typescript@4.2.4 eslint: 8.1.0 + jest-circus: 25.1.0 ndb: 1.1.5 semver: 7.3.5 @@ -13193,6 +13195,30 @@ packages: throat: 6.0.1 dev: true + /jest-circus/25.1.0: + resolution: {integrity: sha512-Axlcr2YMxVarMW4SiZhCFCjNKhdF4xF9AIdltyutQOKyyDT795Kl/fzI95O0l8idE51Npj2wDj5GhrV7uEoEJA==} + engines: {node: '>= 8.3'} + dependencies: + '@babel/traverse': 7.16.3 + '@jest/environment': 25.5.0 + '@jest/test-result': 25.5.0 + '@jest/types': 25.5.0 + chalk: 3.0.0 + co: 4.6.0 + expect: 25.5.0 + is-generator-fn: 2.1.0 + jest-each: 25.5.0 + jest-matcher-utils: 25.5.0 + jest-message-util: 25.5.0 + jest-snapshot: 25.5.1 + jest-util: 25.5.0 + pretty-format: 25.5.0 + stack-utils: 1.0.5 + throat: 5.0.0 + transitivePeerDependencies: + - supports-color + dev: true + /jest-circus/27.3.1: resolution: {integrity: sha512-v1dsM9II6gvXokgqq6Yh2jHCpfg7ZqV4jWY66u7npz24JnhP3NHxI0sKT7+ZMQ7IrOWHYAaeEllOySbDbWsiXw==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}