Add JSON report output to PW tests (#34057)

* Add PW json report when in CI

* Added wait for flaky test

* Used more robust selectors

* Added changelog

.gitignore

@@ -95,6 +95,7 @@ changes.json
 /plugins/woocommerce/e2e/output
 /plugins/woocommerce/e2e/report
 /plugins/woocommerce/e2e/storage
+/plugins/woocommerce/e2e/test-results.json
 
 # Turborepo
 .turbo

plugins/woocommerce/changelog/add-pw-json-report

@@ -0,0 +1,5 @@
+Significance: patch
+Type: add
+Comment: The changes in this PR are for Playwright tests which aren't bundled in woocommerce releases
+
+

plugins/woocommerce/e2e/playwright.config.js

@@ -11,8 +11,15 @@ const config = {
 	workers: 4,
 	reporter: [
 		[ 'list' ],
-		[ 'html', { outputFolder: 'output' } ],
+		[
+			'html',
+			{
+				outputFolder: 'output',
+				open: process.env.CI ? 'never' : 'always',
+			},
+		],
 		[ 'allure-playwright', { outputFolder: 'e2e/allure-results' } ],
+		[ 'json', { outputFile: 'e2e/test-results.json' } ],
 	],
 	use: {
 		screenshot: 'only-on-failure',

plugins/woocommerce/e2e/tests/merchant/order-status-filter.spec.js

@@ -56,6 +56,7 @@ test.describe( 'WooCommerce Orders > Filter Order by Status', () => {
 		await page.goto( 'wp-admin/edit.php?post_type=shop_order' );
 
 		await page.click( 'li.all > a' );
+		await page.waitForLoadState( 'networkidle' );
 		// because tests are running in parallel, we can't know how many orders there
 		// are beyond the ones we created here.
 		for ( let i = 0; i < orderStatus.length; i++ ) {
@@ -72,6 +73,7 @@ test.describe( 'WooCommerce Orders > Filter Order by Status', () => {
 			await page.goto( 'wp-admin/edit.php?post_type=shop_order' );
 
 			await page.click( `li.${ orderStatus[ i ][ 1 ] }` );
+			await page.waitForLoadState( 'networkidle' );
 			const countElements = await page
 				.locator( statusColumnTextSelector )
 				.count();

plugins/woocommerce/e2e/tests/shopper/my-account.spec.js

@@ -22,22 +22,28 @@ test.describe( 'My account page', () => {
 
 		// assert that navigation is visible
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=0' )
+			page.locator( '.woocommerce-MyAccount-navigation-link--dashboard' )
 		).toContainText( 'Dashboard' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=1' )
+			page.locator( '.woocommerce-MyAccount-navigation-link--orders' )
 		).toContainText( 'Orders' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=2' )
+			page.locator( '.woocommerce-MyAccount-navigation-link--downloads' )
 		).toContainText( 'Downloads' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=3' )
+			page.locator(
+				'.woocommerce-MyAccount-navigation-link--edit-address'
+			)
 		).toContainText( 'Addresses' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=4' )
+			page.locator(
+				'.woocommerce-MyAccount-navigation-link--edit-account'
+			)
 		).toContainText( 'Account details' );
 		await expect(
-			page.locator( '.woocommerce-MyAccount-navigation-link >> nth=5' )
+			page.locator(
+				'.woocommerce-MyAccount-navigation-link--customer-logout'
+			)
 		).toContainText( 'Logout' );
 	} );