Merge pull request #31336 from woocommerce/add/jest-retry

Retry failed E2E tests using Jest

packages/js/e2e-environment/README.md

@@ -71,6 +71,20 @@ module.exports = jestConfig;
 
 **NOTE:** Your project's Jest config file is: `tests/e2e/config/jest.config.js`.
 
+### The Jest Object
+
+The E2E environment has the following methods to let us control Jest's overall behavior.
+
+|  Function |  Parameters | Description  |
+|-----------|-------------|--------------|
+| `setupJestRetries` | `retries` | Sets the amount of retries on failed tests 
+
+**NOTE:** The amount of times failed tests are retried can also be set using the `E2E_RETRY_TIMES` environment variable when executing tests. This can be done using the command below: 
+
+```
+E2E_RETRY_TIMES=2 pnpx wc-e2e test:e2e
+```
+
 #### Test Screenshots
 
 The test sequencer provides a screenshot function for test failures. To enable screenshots on test failure use

packages/js/e2e-environment/config/index.js

@@ -3,15 +3,17 @@
  */
 const jestConfig = require( './jest.config' );
 const jestPuppeteerConfig = require( './jest-puppeteer.config' );
+const jestobjectConfig = require('./jest-object.config');
 const {
 	useE2EBabelConfig,
 	useE2EEsLintConfig,
 	useE2EJestConfig,
-	useE2EJestPuppeteerConfig
+	useE2EJestPuppeteerConfig,
 } = require( './use-config' );
 
 module.exports = {
 	jestConfig,
+	...jestobjectConfig,
 	jestPuppeteerConfig,
 	useE2EBabelConfig,
 	useE2EEsLintConfig,

packages/js/e2e-environment/config/jest-object.config.js

@@ -0,0 +1,20 @@
+/**
+ * External Dependencies
+ */
+const { E2E_RETRY_TIMES } = process.env;
+
+const setupJestRetries = ( retries = 2 ) => {
+	const retryTimes = E2E_RETRY_TIMES ? E2E_RETRY_TIMES : retries;
+
+	jest.retryTimes( retryTimes );
+};
+
+// If more methods are added to setupJestObject, it should be include in the readme
+const setupJestObject = () => {
+	setupJestRetries();
+};
+
+module.exports = {
+	setupJestObject,
+	setupJestRetries,
+};

packages/js/e2e-environment/config/jest.config.js

@@ -55,6 +55,9 @@ const combinedConfig = {
 	transformIgnorePatterns: [
 		'node_modules/(?!(woocommerce)/)',
 	],
+
+	testRunner: 'jest-circus/runner',
+
 	roots: [ testSpecs ],
 };
 

packages/js/e2e-environment/package.json

@@ -32,8 +32,8 @@
     "jest": "^25.1.0",
     "jest-each": "25.5.0",
     "jest-puppeteer": "^4.4.0",
-    "request": "^2.88.2",
-    "node-stream-zip": "^1.13.6"
+    "node-stream-zip": "^1.13.6",
+    "request": "^2.88.2"
   },
   "devDependencies": {
     "@babel/cli": "7.12.8",
@@ -42,6 +42,7 @@
     "@babel/preset-env": "7.12.7",
     "@wordpress/eslint-plugin": "7.3.0",
     "eslint": "^8.1.0",
+    "jest-circus": "25.1.0",
     "ndb": "^1.1.5",
     "semver": "^7.3.2"
   },

packages/js/e2e-environment/src/setup/jest.setup.js

@@ -9,6 +9,7 @@ import {
 	setBrowserViewport,
 } from '@wordpress/e2e-test-utils';
 import { consoleShouldSuppress, addConsoleSuppression } from '../../utils';
+import { setupJestRetries } from '../../config/jest-object.config';
 
 /**
  * Array of page event tuples of [ eventName, handler ].
@@ -175,6 +176,7 @@ beforeAll( async () => {
 	capturePageEventsForTearDown();
 	enablePageDialogAccept();
 	observeConsoleLogging();
+	setupJestRetries();
 } );
 
 afterEach( async () => {

packages/js/e2e-utils/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Unreleased
 
+## Fixed
+
+- Identified the default product category using `slug == 'uncategorized'` in `deleteAllProductCategories`
+
 ## Changes
 
 - Removed `page.waitForNavigation()` from `shopper.logout()`

packages/js/e2e-utils/src/flows/with-rest-api.js

@@ -138,7 +138,7 @@ export const withRestApi = {
 		if ( productCategories.data && productCategories.data.length ) {
 			for ( let c = 0; c < productCategories.data.length; c++ ) {
 				// The default `uncategorized` category can't be deleted
-				if ( productCategories.data[c].id == 0 ) {
+				if ( productCategories.data[c].slug == 'uncategorized' ) {
 					continue;
 				}
 				const response = await client.delete( productCategoriesPath + `/${productCategories.data[c].id}?force=true` );

plugins/woocommerce/tests/e2e/README.md

@@ -186,6 +186,15 @@ For example:
 - `PUPPETEER_SLOWMO=10` - will run tests faster
 - `PUPPETEER_SLOWMO=70` - will run tests slower
 
+### How to retry failed tests
+
+Sometimes tests may fail for different reasons such as network issues, or lost connection. To mitigate against test flakiess, failed tests are rerun up to 3 times before being marked as failed. The amount of retry attempts can be adjusted by passing the `E2E_RETRY_TIMES` variable when running tests. For example:
+
+```
+cd plugins/woocommerce
+E2E_RETRY_TIMES=2 pnpx wc-e2e test:e2e
+```
+
 ### How to run tests in debug mode
 
 Tests run in headless mode by default. While writing tests it may be useful to have the debugger loaded while running a test in non-headless mode. To run tests in debug mode:

plugins/woocommerce/tests/e2e/config/jest.setup.js

@@ -7,7 +7,7 @@ import {
 
 const config = require( 'config' );
 const { HTTPClientFactory } = require( '@woocommerce/api' );
-const { addConsoleSuppression, updateReadyPageStatus } = require( '@woocommerce/e2e-environment' );
+const { addConsoleSuppression, updateReadyPageStatus, setupJestRetries } = require( '@woocommerce/e2e-environment' );
 const { DEFAULT_TIMEOUT_OVERRIDE } = process.env;
 
 // @todo: remove this once https://github.com/woocommerce/woocommerce-admin/issues/6992 has been addressed
@@ -40,6 +40,8 @@ async function trashExistingPosts() {
 // each other's side-effects.
 beforeAll(async () => {
 
+	setupJestRetries();
+
 	if ( DEFAULT_TIMEOUT_OVERRIDE ) {
 		page.setDefaultNavigationTimeout( DEFAULT_TIMEOUT_OVERRIDE );
 		page.setDefaultTimeout( DEFAULT_TIMEOUT_OVERRIDE );

pnpm-lock.yaml

@@ -125,6 +125,7 @@ importers:
       commander: 4.1.1
       eslint: ^8.1.0
       jest: ^25.1.0
+      jest-circus: 25.1.0
       jest-each: 25.5.0
       jest-puppeteer: ^4.4.0
       ndb: ^1.1.5
@@ -152,6 +153,7 @@ importers:
       '@babel/preset-env': 7.12.7_@babel+core@7.12.9
       '@wordpress/eslint-plugin': 7.3.0_eslint@8.1.0+typescript@4.2.4
       eslint: 8.1.0
+      jest-circus: 25.1.0
       ndb: 1.1.5
       semver: 7.3.5
 
@@ -13193,6 +13195,30 @@ packages:
       throat: 6.0.1
     dev: true
 
+  /jest-circus/25.1.0:
+    resolution: {integrity: sha512-Axlcr2YMxVarMW4SiZhCFCjNKhdF4xF9AIdltyutQOKyyDT795Kl/fzI95O0l8idE51Npj2wDj5GhrV7uEoEJA==}
+    engines: {node: '>= 8.3'}
+    dependencies:
+      '@babel/traverse': 7.16.3
+      '@jest/environment': 25.5.0
+      '@jest/test-result': 25.5.0
+      '@jest/types': 25.5.0
+      chalk: 3.0.0
+      co: 4.6.0
+      expect: 25.5.0
+      is-generator-fn: 2.1.0
+      jest-each: 25.5.0
+      jest-matcher-utils: 25.5.0
+      jest-message-util: 25.5.0
+      jest-snapshot: 25.5.1
+      jest-util: 25.5.0
+      pretty-format: 25.5.0
+      stack-utils: 1.0.5
+      throat: 5.0.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: true
+
   /jest-circus/27.3.1:
     resolution: {integrity: sha512-v1dsM9II6gvXokgqq6Yh2jHCpfg7ZqV4jWY66u7npz24JnhP3NHxI0sKT7+ZMQ7IrOWHYAaeEllOySbDbWsiXw==}
     engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}