From 7e3b3ee94c12fa2f4ac8f97cce451d823d17f510 Mon Sep 17 00:00:00 2001
From: Jamel Noel Reid <MrJnrman@users.noreply.github.com>
Date: Mon, 25 Jul 2022 19:44:11 +0200
Subject: [PATCH] Add JSON report output to PW tests (#34057)

* Add PW json report when in CI

* Added wait for flaky test

* Used more robust selectors

* Added changelog
---
 .gitignore                                     |  1 +
 .../woocommerce/changelog/add-pw-json-report   |  5 +++++
 plugins/woocommerce/e2e/playwright.config.js   |  9 ++++++++-
 .../tests/merchant/order-status-filter.spec.js |  2 ++
 .../e2e/tests/shopper/my-account.spec.js       | 18 ++++++++++++------
 5 files changed, 28 insertions(+), 7 deletions(-)
 create mode 100644 plugins/woocommerce/changelog/add-pw-json-report

diff --git a/.gitignore b/.gitignore
index 6ebfd7d97ac..1fd53931727 100644
--- a/.gitignore
+++ b/.gitignore
@@ -95,6 +95,7 @@ changes.json
 /plugins/woocommerce/e2e/output
 /plugins/woocommerce/e2e/report
 /plugins/woocommerce/e2e/storage
+/plugins/woocommerce/e2e/test-results.json
 
 # Turborepo
 .turbo
diff --git a/plugins/woocommerce/changelog/add-pw-json-report b/plugins/woocommerce/changelog/add-pw-json-report
new file mode 100644
index 00000000000..bb37ad228a3
--- /dev/null
+++ b/plugins/woocommerce/changelog/add-pw-json-report
@@ -0,0 +1,5 @@
+Significance: patch
+Type: add
+Comment: The changes in this PR are for Playwright tests which aren't bundled in woocommerce releases
+
+
diff --git a/plugins/woocommerce/e2e/playwright.config.js b/plugins/woocommerce/e2e/playwright.config.js
index 6b9c30c240f..1f3042efff0 100644
--- a/plugins/woocommerce/e2e/playwright.config.js
+++ b/plugins/woocommerce/e2e/playwright.config.js
@@ -11,8 +11,15 @@ const config = {
 	workers: 4,
 	reporter: [
 		[ 'list' ],
-		[ 'html', { outputFolder: 'output' } ],
+		[
+			'html',
+			{
+				outputFolder: 'output',
+				open: process.env.CI ? 'never' : 'always',
+			},
+		],
 		[ 'allure-playwright', { outputFolder: 'e2e/allure-results' } ],
+		[ 'json', { outputFile: 'e2e/test-results.json' } ],
 	],
 	use: {
 		screenshot: 'only-on-failure',
diff --git a/plugins/woocommerce/e2e/tests/merchant/order-status-filter.spec.js b/plugins/woocommerce/e2e/tests/merchant/order-status-filter.spec.js
index 53677a38212..589a3bf08b9 100644
--- a/plugins/woocommerce/e2e/tests/merchant/order-status-filter.spec.js
+++ b/plugins/woocommerce/e2e/tests/merchant/order-status-filter.spec.js
@@ -56,6 +56,7 @@ test.describe( 'WooCommerce Orders > Filter Order by Status', () => {
 		await page.goto( 'wp-admin/edit.php?post_type=shop_order' );
 
 		await page.click( 'li.all > a' );
+		await page.waitForLoadState( 'networkidle' );
 		// because tests are running in parallel, we can't know how many orders there
 		// are beyond the ones we created here.
 		for ( let i = 0; i < orderStatus.length; i++ ) {
@@ -72,6 +73,7 @@ test.describe( 'WooCommerce Orders > Filter Order by Status', () => {
 			await page.goto( 'wp-admin/edit.php?post_type=shop_order' );
 
 			await page.click( `li.${ orderStatus[ i ][ 1 ] }` );
+			await page.waitForLoadState( 'networkidle' );
 			const countElements = await page
 				.locator( statusColumnTextSelector )
 				.count();
diff --git a/plugins/woocommerce/e2e/tests/shopper/my-account.spec.js b/plugins/woocommerce/e2e/tests/shopper/my-account.spec.js
index 77323ca37e2..72a4b61fe35 100644
--- a/plugins/woocommerce/e2e/tests/shopper/my-account.spec.js
+++ b/plugins/woocommerce/e2e/tests/shopper/my-account.spec.js
@@ -22,22 +22,28 @@ test.describe( 'My account page', () => {
 
 		// assert that navigation is visible
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=0' )
+			page.locator( '.woocommerce-MyAccount-navigation-link--dashboard' )
 		).toContainText( 'Dashboard' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=1' )
+			page.locator( '.woocommerce-MyAccount-navigation-link--orders' )
 		).toContainText( 'Orders' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=2' )
+			page.locator( '.woocommerce-MyAccount-navigation-link--downloads' )
 		).toContainText( 'Downloads' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=3' )
+			page.locator(
+				'.woocommerce-MyAccount-navigation-link--edit-address'
+			)
 		).toContainText( 'Addresses' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=4' )
+			page.locator(
+				'.woocommerce-MyAccount-navigation-link--edit-account'
+			)
 		).toContainText( 'Account details' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=5' )
+			page.locator(
+				'.woocommerce-MyAccount-navigation-link--customer-logout'
+			)
 		).toContainText( 'Logout' );
 	} );