Merge pull request #24479 from woocommerce/new/puppeteer-architecture

Implement Puppeteer architecture & add basic test

.eslintrc

@@ -22,6 +22,13 @@
 		"max-len": [ 2, { "code": 140 } ],
 		"no-console": 1
 	},
+	"plugins": [
+		"jest"
+	],
+	"extends": [
+		"plugin:jest/recommended"
+	],
+	"parser": "babel-eslint",
 	"parserOptions": {
 		"ecmaVersion": 8,
 		"ecmaFeatures": {
@@ -29,11 +36,5 @@
 			"experimentalObjectRestSpread": true,
 			"jsx": true
 		}
-	},
+	}
-	"plugins": [
-		"jest"
-	],
-	"extends": [
-		"plugin:jest/recommended"
-	]
 }

.gitignore

@@ -44,9 +44,6 @@ tests/cli/vendor
 /tests/e2e-tests/config/local-*.json
 /tests/e2e-tests/config/local.json
 
-# Screenshot tests
-/__image_snapshots__/
-
 # Logs
 /logs
 

.travis.yml

@@ -35,14 +35,6 @@ matrix:
       apt:
         packages:
           - nginx
-  - name: "Visual regression tests"
-    php: 7.3
-    env: WP_VERSION=latest WP_MULTISITE=0 RUN_VIS_REGRESSION=1
-    addons:
-      chrome: stable
-      apt:
-        packages:
-          - nginx
   - name: "Unit tests code coverage"
     php: 7.3
     env: WP_VERSION=latest WP_MULTISITE=0 RUN_CODE_COVERAGE=1
@@ -74,7 +66,6 @@ script:
   - bash tests/bin/phpunit.sh
   - bash tests/bin/phpcs.sh
   - travis_retry bash tests/bin/run-e2e-CI.sh
-  - bash tests/bin/run-vis-regression.sh
 
 after_script:
   - bash tests/bin/travis.sh after

babel.config.js

@@ -0,0 +1,12 @@
+module.exports = {
+	presets: [
+		[
+			'@babel/preset-env',
+			{
+				targets: {
+					node: 'current',
+				},
+			},
+		],
+	],
+};

package.json

@@ -13,24 +13,23 @@
     "build": "grunt",
     "build-watch": "grunt watch",
     "lint:js": "eslint assets/js --ext=js",
-    "test": "cross-env NODE_CONFIG_DIR='./tests/e2e-tests/config' BABEL_ENV=commonjs mocha \"tests/e2e-tests\" --require babel-register --recursive",
+    "test:e2e": "./tests/bin/e2e-test-integration.js",
-    "test:grep": "cross-env NODE_CONFIG_DIR='./tests/e2e-tests/config' BABEL_ENV=commonjs mocha \"tests/e2e-tests\" --require babel-register --grep ",
-    "test:single": "cross-env NODE_CONFIG_DIR='./tests/e2e-tests/config' BABEL_ENV=commonjs mocha --require babel-register",
-    "test:vis-regression": "cross-env NODE_CONFIG_DIR='./tests/screenshot-tests/config' NODE_ENV=development JEST_PUPPETEER_CONFIG='./tests/screenshot-tests/config/jest-puppeteer.config.js' jest --config='./tests/screenshot-tests/config/jest.config.js' --maxWorkers=1 --rootDir=./",
     "makepot": "grunt makepot",
     "git:update-hooks": "rm -r .git/hooks && mkdir -p .git/hooks && node ./node_modules/husky/husky.js install"
   },
   "devDependencies": {
+    "@wordpress/e2e-test-utils": "^2.2.0",
     "autoprefixer": "9.6.1",
-    "babel": "6.23.0",
+    "@babel/cli": "^7.5.5",
-    "babel-cli": "6.26.0",
+    "@babel/core": "^7.5.5",
+    "@babel/polyfill": "^7.4.4",
+    "@babel/preset-env": "^7.5.5",
+    "@babel/register": "^7.5.5",
     "babel-eslint": "10.0.2",
-    "babel-plugin-add-module-exports": "1.0.2",
-    "babel-preset-es2015": "6.24.1",
-    "babel-preset-stage-2": "6.24.1",
     "chai": "4.2.0",
     "chai-as-promised": "7.1.1",
     "chromedriver": "75.1.0",
+    "commander": "^3.0.0",
     "config": "3.2.2",
     "cross-env": "5.2.0",
     "eslint": "6.1.0",
@@ -58,7 +57,6 @@
     "husky": "3.0.1",
     "istanbul": "1.0.0-alpha.2",
     "jest": "24.8.0",
-    "jest-image-snapshot": "2.9.0",
     "jest-puppeteer": "4.3.0",
     "lint-staged": "9.2.0",
     "mocha": "6.2.0",

tests/bin/e2e-test-integration.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+
+const { spawn } = require( 'child_process' );
+const program = require( 'commander' );
+
+program
+	.usage( '<file ...> [options]' )
+	.option( '--dev', 'Development mode' )
+	.parse( process.argv );
+
+const testEnvVars = {
+	NODE_ENV: 'test:e2e',
+	JEST_PUPPETEER_CONFIG: 'tests/e2e-tests/config/jest-puppeteer.config.js',
+	NODE_CONFIG_DIR: 'tests/e2e-tests/config',
+};
+
+if ( program.dev ) {
+	testEnvVars.JEST_PUPPETEER_CONFIG = 'tests/e2e-tests/config/jest-puppeteer.dev.config.js';
+}
+
+const envVars = Object.assign( {}, process.env, testEnvVars );
+
+spawn(
+	'jest',
+	[
+		'--maxWorkers=1',
+		'--config=tests/e2e-tests/config/jest.config.js',
+		'--rootDir=./',
+		program.args,
+	],
+	{
+		stdio: 'inherit',
+		env: envVars,
+	}
+);

tests/bin/install.sh

@@ -150,7 +150,7 @@ install_db() {
 
 install_e2e_site() {
 
-	if [[ ${RUN_E2E} == 1 || ${RUN_VIS_REGRESSION} == 1 ]]; then
+	if [[ ${RUN_E2E} == 1 ]]; then
 
 		# Script Variables
 		CONFIG_DIR="./tests/e2e-tests/config/travis"

tests/bin/phpunit.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-if [[ ${RUN_PHPCS} == 1 ]] || [[ ${RUN_E2E} == 1 ]] || [[ ${RUN_VIS_REGRESSION} == 1 ]]; then
+if [[ ${RUN_PHPCS} == 1 ]] || [[ ${RUN_E2E} == 1 ]]; then
 	exit
 fi
 

tests/bin/run-vis-regression.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-if [[ ${RUN_VIS_REGRESSION} == 1 ]]; then
-
-	WP_SITE_URL="http://localhost:8080"
-
-	# Setup enviromental variables
-	export BASE_URL="$WP_SITE_URL"
-
-	# Run the tests
-	npm run test:vis-regression
-fi

tests/bin/travis.sh

@@ -10,12 +10,7 @@ if [ $1 == 'after' ]; then
 		php ocular.phar code-coverage:upload --format=php-clover coverage.clover
 	fi
 
-	if [[ ${RUN_VIS_REGRESSION} == 1 ]]; then
+	if [[ ${RUN_E2E} == 1 && $(ls -A $TRAVIS_BUILD_DIR/screenshots) ]]; then
-		# copy diff output for failed tests to the screenshots directory to uploading.
-		cp /tests/visual-regression/__image_snapshots__/__diff_output__/ $TRAVIS_BUILD_DIR/screenshots/
-	fi
-
-	if [[ ( ${RUN_E2E} == 1 || ${RUN_VIS_REGRESSION} == 1 ) && $(ls -A $TRAVIS_BUILD_DIR/screenshots) ]]; then
 		if [[ -z "${ARTIFACTS_KEY}" ]]; then
 			echo "Screenshots were not uploaded. Please run the e2e tests locally to see failures."
 		else

tests/e2e-tests/README.md

@@ -0,0 +1,124 @@
+# WooCommerce End to End Tests
+
+Automated end-to-end tests for WooCommerce.
+
+## Table of contents
+
+- [Pre-requisites](#pre-requisites)
+  - [Install NodeJS](#install-nodejs)
+  - [Install dependencies](#install-dependencies)
+  - [Configuration](#configuration)
+      - [Test Configuration](#test-configuration)
+      - [Environment Variables](#environment-variables)
+- [Running tests](#running-tests)
+  - [How to run tests](#how-to-run-tests) 
+- [Writing tests](#writing-tests) 
+
+## Pre-requisites
+
+### Install NodeJS
+
+```bash
+brew install node #MacOS
+```
+
+### Install dependencies
+
+```bash
+npm install
+```
+
+### Configuration
+
+#### Test Configuration
+
+The tests use environment variables to specify login test data needed to run tests. 
+
+To login to the site, `loginUser()` utility function of [`e2e-test-utils`](https://github.com/WordPress/gutenberg/tree/master/packages/e2e-test-utils) package is being used. The function relies on the following [`config.js`](https://github.com/WordPress/gutenberg/blob/master/packages/e2e-test-utils/src/shared/config.js) file to specify base URL, Admin user details and Test user details (could be different from Admin. For example, customer):
+
+```
+const WP_ADMIN_USER = {
+ 	username: 'admin',
+ 	password: 'password',
+ };
+ 
+ const {
+ 	WP_USERNAME = WP_ADMIN_USER.username,
+ 	WP_PASSWORD = WP_ADMIN_USER.password,
+ 	WP_BASE_URL = 'http://localhost:8889',
+ } = process.env;
+ 
+ export {
+ 	WP_ADMIN_USER,
+ 	WP_USERNAME,
+ 	WP_PASSWORD,
+ 	WP_BASE_URL,
+ };
+```    
+
+As per above, create an Admin user on the site and set its username and password:
+
+- username: `admin`
+- password: `password`
+
+Specify base URL and Test user details using environment variables. 
+
+#### Environment variables
+
+Set environmental variables as shown below. Note that you don't need to add the trailing slash ('/') at the end of the site URL:
+
+- `export WP_BASE_URL={your site URL}`
+- `export WP_USERNAME={your Test user username}`
+- `export WP_PASSWORD={your Test user password}`
+
+You can unset the variables when you are done:
+
+- `unset WP_BASE_URL`
+- `unset WP_USERNAME`
+- `unset WP_PASSWORD`
+
+## Running tests
+
+### How to run tests
+
+To run e2e tests use the following command:
+
+```bash
+npm run test:e2e
+```
+
+Tests are being run headless by default. However, sometimes it's useful to observe the browser while running tests. To do so, `Development mode` can be enabled by passing `--dev` flag to the `test:e2e` script in `./package.json` file as follows:
+                                    
+```bash
+"test:e2e": "./tests/bin/e2e-test-integration.js --dev",
+```
+
+Once done, start tests as usual by running `npm run test:e2e`. 
+
+The `Development mode` also enables SlowMo mode. SlowMo slows down Puppeteer’s operations so we can better see what is happening in the browser. You can adjust the SlowMo value by editing `PUPPETEER_SLOWMO` variable in `./tests/e2e-tests/config/jest-puppeteer.dev.config.js` file. The default `PUPPETEER_SLOWMO=50` means test actions will be slowed down by 50 milliseconds.
+
+To run an individual test, use the direct path to the spec. For example:
+
+```bash
+npm run test:e2e ./tests/e2e-tests/specs/activate-woocommerce.test.js
+``` 
+
+You can also provide the base URL, Test username and Test password like this:
+
+```bash
+WP_BASE_URL={your site URL} WP_USERNAME={your Test user username} WP_PASSWORD={your Test user password} npm run test:e2e
+```
+
+## Writing tests
+
+We use the following tools to write e2e tests:
+
+- [Puppeteer](https://github.com/GoogleChrome/puppeteer) – a Node library which provides a high-level API to control Chrome or Chromium over the DevTools Protocol
+- [jest-puppeteer](https://github.com/smooth-code/jest-puppeteer) – provides all required configuration to run tests using Puppeteer
+- [expect-puppeteer](https://github.com/smooth-code/jest-puppeteer/tree/master/packages/expect-puppeteer) – assertion library for Puppeteer
+
+Tests are kept in `tests/e2e-tests/specs` folder. 
+
+The following packages are being used to write tests:
+
+- `e2e-test-utils` - End-To-End (E2E) test utils for WordPress. You can find the full list of utils [here](https://github.com/WordPress/gutenberg/tree/master/packages/e2e-test-utils). 

tests/e2e-tests/config/custom-environment-variables.json

@@ -1,3 +1,3 @@
 {
-	"url": "BASE_URL"
+  "url": "BASE_URL"
 }

tests/e2e-tests/config/default.json

@@ -1,15 +1,3 @@
 {
-	"url": "http://example.com",
+  "url": "https://example.com/"
-	"users": {
-		"admin": {
-			"username": "",
-			"password": ""
-		},
-		"customer": {
-			"username": "",
-			"password": ""
-		}
-	},
-	"startBrowserTimeoutMs": 30000,
-	"mochaTimeoutMs": 120000
 }

tests/e2e-tests/config/jest-puppeteer.config.js

@@ -0,0 +1,6 @@
+/** @format */
+
+module.exports = {
+	// Required for the logged out and logged in tests so they don't share app state/token.
+	browserContext: 'incognito',
+};

tests/e2e-tests/config/jest-puppeteer.dev.config.js

@@ -2,8 +2,8 @@
 
 module.exports = {
 	launch: {
-		slowMo: false,
+		slowMo: process.env.PUPPETEER_SLOWMO ? false : 50,
-		headless: true,
+		headless: process.env.PUPPETEER_HEADLESS || false,
 		ignoreHTTPSErrors: true,
 		args: [
 			'--window-size=1920,1080',
@@ -14,8 +14,7 @@ module.exports = {
 			width: 1280,
 			height: 800,
 		},
+		// Required for the logged out and logged in tests so they don't share app state/token.
 		browserContext: 'incognito',
-		// This will pipe browser's console to node.
-		dumpio: true,
 	}
 };

tests/e2e-tests/config/jest.config.js

@@ -15,14 +15,14 @@ module.exports = {
 	preset: 'jest-puppeteer',
 
 	// Where to look for test files
-	roots: [ '<rootDir>/tests/screenshot-tests' ],
+	roots: [ '<rootDir>/tests/e2e-tests/specs' ],
 
 	//setupFiles: [ '<rootDir>/.node_modules/regenerator-runtime/runtime' ],
 
 	// A list of paths to modules that run some code to configure or set up the testing framework
 	// before each test
 	setupFilesAfterEnv: [
-		'<rootDir>/tests/screenshot-tests/config/jest.setup.js',
+		'<rootDir>/tests/e2e-tests/config/jest.setup.js',
 		'expect-puppeteer',
 	],
 

tests/e2e-tests/config/jest.setup.js

@@ -0,0 +1,7 @@
+/** format */
+
+/**
+ * Increase the default timeout to 10s
+ */
+let jestTimeoutInMilliSeconds = 10000;
+jest.setTimeout( jestTimeoutInMilliSeconds );

tests/e2e-tests/config/local-sample.json

@@ -1,13 +0,0 @@
-{
-  "url": "http://local.wordpress.dev/",
-  "users": {
-    "admin": {
-      "username": "admin",
-      "password": "password"
-    },
-    "customer": {
-      "username": "Customer",
-      "password": "password"
-    }
-  }
-}

tests/e2e-tests/specs/activate-woocommerce.test.js

@@ -0,0 +1,34 @@
+/**
+ * WordPress dependencies
+ */
+import { activatePlugin } from "@wordpress/e2e-test-utils";
+
+/**
+ * activatePlugin( slug ) - Activates an installed plugin.
+ *
+ * @param {string} slug Plugin slug.
+ *
+ * Within the activatePlugin() function:
+ *
+ * 1) Switches the current user to the admin user (if the user
+ * running the test is not already the admin user).
+ *
+ * 2) Visits admin page; if user is not logged in then it logging in it first, then visits admin page.
+ *
+ * 3) Checks if plugin is activated. If not, activates the plugin.
+ *
+ * 4) Switches the current user to whichever user we should be
+ * running the tests as (if we're not already that user).
+ */
+
+describe( 'Can login and make sure WooCommerce plugin is activated', () => {
+
+	test( 'Can activate WooCommerce plugin if it is deactivated' , async () => {
+		try {
+			await activatePlugin( 'woocommerce' );
+		} catch ( error ) {
+			await activatePlugin( 'woocommerce' );
+		}
+	});
+
+} );

tests/screenshot-tests/config/jest.setup.js

@@ -1,13 +0,0 @@
-/** format */
-
-/**
- * Increase the default timeout to 10s
- */
-let jestTimeoutInMilliSeconds = 10000;
-jest.setTimeout( jestTimeoutInMilliSeconds );
-
-/**
- * Extend expect to have the toMatchImageSnapshop function
- */
-const { toMatchImageSnapshot } = require('jest-image-snapshot');
-expect.extend({ toMatchImageSnapshot });