[e2e tests] Fix blocks tests failing with Gutenberg active (#46842)

* Fix plugin slug for reporting

* Comment out setting of woocommerce_coming_soon

* Add canvas utility

* Update create-cart-block.spec.js to use canvas instead of page

* Update create-checkout-block.spec.js to use canvas instead of page

* Extract common editor actions into editor utils

* Add changelog

* Update post and page creation tests to use canvas

* Update create-woocommerce-blocks.spec.js to use canvas

* Activate Gutenberg to test changes

* Remove plugins listing from env setup script

* Fix eslint errors

* Remove redundant env setup script call

* Install Gutenberg for test in CI

* Test with Gutenberg

* Remove Gutenberg

* Fix block editor variable product test

* Revert "Fix block editor variable product test"

This reverts commit cd004530a7.

---------

Co-authored-by: Jon Lane <jon.lane@automattic.com>

.github/workflows/smoke-test-daily.yml

@@ -406,7 +406,7 @@ jobs:
                     - plugin: 'Gutenberg'
                       slug: gutenberg
                     - plugin: 'Gutenberg - Nightly'
-                      slug: gutenberg
+                      slug: gutenberg-nightly
         steps:
             - name: Download test report artifact
               uses: actions/download-artifact@v3

plugins/woocommerce/changelog/e2e-fix-page-title-in-editor-with-gutenberg

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+E2E tests: fix tests that are failing with Gutenberg active

plugins/woocommerce/package.json

@@ -28,8 +28,8 @@
 		"env:restart": "pnpm wp-env destroy && pnpm wp-env start --update",
 		"env:start": "pnpm wp-env start",
 		"env:stop": "pnpm wp-env stop",
-		"env:test": "WP_ENV_LIFECYCLE_SCRIPT_AFTER_START='./tests/e2e-pw/bin/test-env-setup.sh' && pnpm env:dev && pnpm playwright install chromium",
-		"env:test:cot": "WP_ENV_LIFECYCLE_SCRIPT_AFTER_START='ENABLE_HPOS=1 ./tests/e2e-pw/bin/test-env-setup.sh' && ENABLE_HPOS=1 pnpm env:dev",
+		"env:test": "pnpm env:dev && pnpm playwright install chromium",
+		"env:test:cot": "ENABLE_HPOS=1 pnpm env:test",
 		"env:perf:install-k6": "curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1",
 		"env:perf": "pnpm env:dev && pnpm env:performance-init && pnpm env:perf:install-k6",
 		"preinstall": "npx only-allow pnpm",

plugins/woocommerce/tests/e2e-pw/bin/test-env-setup.sh

@@ -40,4 +40,3 @@ fi
 
 echo -e 'Upload test images \n'
 wp-env run tests-cli wp media import './test-data/images/image-01.png' './test-data/images/image-02.png' './test-data/images/image-03.png'
-

plugins/woocommerce/tests/e2e-pw/global-setup.js

@@ -1,10 +1,9 @@
-const { chromium, expect, request } = require( '@playwright/test' );
+const { chromium, expect } = require( '@playwright/test' );
 const { admin, customer } = require( './test-data/data' );
 const fs = require( 'fs' );
 const { site } = require( './utils' );
 const wcApi = require( '@woocommerce/woocommerce-rest-api' ).default;
 const { ENABLE_HPOS } = process.env;
-const { setOption } = require( './utils/options' );
 
 /**
  * @param {import('@playwright/test').FullConfig} config
@@ -57,9 +56,6 @@ module.exports = async ( config ) => {
 	const adminPage = await adminContext.newPage();
 	const customerPage = await customerContext.newPage();
 
-	// Ensure live mode state (coming soon = no)
-	await setOption( request, baseURL, 'woocommerce_coming_soon', 'no' );
-
 	// Sign in as admin user and save state
 	const adminRetries = 5;
 	for ( let i = 0; i < adminRetries; i++ ) {

plugins/woocommerce/tests/e2e-pw/tests/merchant/create-cart-block.spec.js

@@ -1,5 +1,10 @@
 const { test, expect } = require( '@playwright/test' );
-const { disableWelcomeModal } = require( '../../utils/editor' );
+const {
+	goToPageEditor,
+	fillPageTitle,
+	insertBlock,
+	transformIntoBlocks,
+} = require( '../../utils/editor' );
 
 const transformedCartBlockTitle = `Transformed Cart ${ Date.now() }`;
 const transformedCartBlockSlug = transformedCartBlockTitle
@@ -10,40 +15,11 @@ test.describe( 'Transform Classic Cart To Cart Block', () => {
 	test.use( { storageState: process.env.ADMINSTATE } );
 
 	test( 'can transform classic cart to cart block', async ( { page } ) => {
-		// go to create a new page
-		await page.goto( 'wp-admin/post-new.php?post_type=page' );
+		await goToPageEditor( { page } );
 
-		await disableWelcomeModal( { page } );
-
-		// fill page title
-		await page
-			.getByRole( 'textbox', { name: 'Add title' } )
-			.fill( transformedCartBlockTitle );
-
-		// add classic cart block
-		await page.getByRole( 'textbox', { name: 'Add title' } ).click();
-		await page.getByLabel( 'Add block' ).click();
-		await page
-			.getByPlaceholder( 'Search', { exact: true } )
-			.fill( 'classic cart' );
-		await page
-			.getByRole( 'option' )
-			.filter( { hasText: 'Classic Cart' } )
-			.click();
-
-		// transform into blocks
-		await expect(
-			page.locator(
-				'.wp-block-woocommerce-classic-shortcode__placeholder-copy'
-			)
-		).toBeVisible();
-		await page
-			.getByRole( 'button' )
-			.filter( { hasText: 'Transform into blocks' } )
-			.click();
-		await expect( page.getByLabel( 'Dismiss this notice' ) ).toContainText(
-			'Classic shortcode transformed to blocks.'
-		);
+		await fillPageTitle( page, transformedCartBlockTitle );
+		await insertBlock( page, 'Classic Cart' );
+		await transformIntoBlocks( page );
 
 		// save and publish the page
 		await page

plugins/woocommerce/tests/e2e-pw/tests/merchant/create-checkout-block.spec.js

@@ -1,5 +1,11 @@
 const { test, expect } = require( '@playwright/test' );
-const { disableWelcomeModal } = require( '../../utils/editor' );
+const {
+	goToPageEditor,
+	getCanvas,
+	fillPageTitle,
+	insertBlock,
+	transformIntoBlocks,
+} = require( '../../utils/editor' );
 const wcApi = require( '@woocommerce/woocommerce-rest-api' ).default;
 
 const transformedCheckoutBlockTitle = `Transformed Checkout ${ Date.now() }`;
@@ -81,44 +87,18 @@ test.describe( 'Transform Classic Checkout To Checkout Block', () => {
 			version: 'wc/v3',
 		} );
 
-		// go to create a new page
-		await page.goto( 'wp-admin/post-new.php?post_type=page' );
+		await goToPageEditor( { page } );
 
-		await disableWelcomeModal( { page } );
+		await fillPageTitle( page, transformedCheckoutBlockTitle );
+		await insertBlock( page, 'Classic Checkout' );
+		await transformIntoBlocks( page );
 
-		// fill page title
-		await page
-			.getByRole( 'textbox', { name: 'Add title' } )
-			.fill( transformedCheckoutBlockTitle );
+		// When Gutenberg is active, the canvas is in an iframe
+		let canvas = await getCanvas( page );
 
-		// add classic checkout block
-		await page.getByRole( 'textbox', { name: 'Add title' } ).click();
-		await page.getByLabel( 'Add block' ).click();
-		await page
-			.getByPlaceholder( 'Search', { exact: true } )
-			.fill( 'classic checkout' );
-		await page
-			.getByRole( 'option' )
-			.filter( { hasText: 'Classic Checkout' } )
-			.click();
-
-		// transform into blocks
-		await expect(
-			page.locator(
-				'.wp-block-woocommerce-classic-shortcode__placeholder-copy'
-			)
-		).toBeVisible();
-		await page
-			.getByRole( 'button' )
-			.filter( { hasText: 'Transform into blocks' } )
-			.click();
-		await expect( page.getByLabel( 'Dismiss this notice' ) ).toContainText(
-			'Classic shortcode transformed to blocks.'
-		);
-
-		// set terms & conditions and privacy policy as mandatory option
-		await page.locator( '.wc-block-checkout__terms' ).click();
-		await page.getByLabel( 'Require checkbox' ).click();
+		// Activate the terms and conditions checkbox
+		await canvas.getByLabel( 'Block: Terms and Conditions' ).click();
+		await page.getByLabel( 'Require checkbox' ).check();
 
 		// save and publish the page
 		await page
@@ -138,17 +118,16 @@ test.describe( 'Transform Classic Checkout To Checkout Block', () => {
 		} );
 		await page.reload();
 
-		// verify that enabled payment options are in the block
+		// Mandatory to wait for the editor content, to ensure the iframe is loaded (if Gutenberg is active)
+		await expect( page.getByLabel( 'Editor content' ) ).toBeVisible();
+
+		// Get the canvas again after the page reload
+		canvas = await getCanvas( page );
+
 		await expect(
-			page.locator(
-				'#radio-control-wc-payment-method-options-bacs__label'
-			)
-		).toContainText( 'Direct bank transfer' );
-		await expect(
-			page.locator(
-				'#radio-control-wc-payment-method-options-cod__label'
-			)
-		).toContainText( 'Cash on delivery' );
+			canvas.getByText( 'Direct bank transfer' )
+		).toBeVisible();
+		await expect( canvas.getByText( 'Cash on delivery' ) ).toBeVisible();
 
 		// add additional shipping methods after page creation
 		await api.post( `shipping/zones/${ shippingZoneId }/methods`, {

plugins/woocommerce/tests/e2e-pw/tests/merchant/create-page.spec.js

@@ -1,6 +1,10 @@
 const { test, expect, request } = require( '@playwright/test' );
 const { admin } = require( '../../test-data/data' );
-const { goToPageEditor } = require( '../../utils/editor' );
+const {
+	goToPageEditor,
+	fillPageTitle,
+	getCanvas,
+} = require( '../../utils/editor' );
 
 const pageTitle = `Page-${ new Date().getTime().toString() }`;
 
@@ -35,13 +39,15 @@ test.describe( 'Can create a new page', () => {
 	test( 'can create new page', async ( { page } ) => {
 		await goToPageEditor( { page } );
 
-		await page
-			.getByRole( 'textbox', { name: 'Add Title' } )
-			.fill( pageTitle );
+		await fillPageTitle( page, pageTitle );
 
-		await page.getByRole( 'button', { name: 'Add default block' } ).click();
+		const canvas = await getCanvas( page );
 
-		await page
+		await canvas
+			.getByRole( 'button', { name: 'Add default block' } )
+			.click();
+
+		await canvas
 			.getByRole( 'document', {
 				name: 'Empty block; start writing or type forward slash to choose a block',
 			} )

plugins/woocommerce/tests/e2e-pw/tests/merchant/create-post.spec.js

@@ -1,6 +1,10 @@
 const { test, expect, request } = require( '@playwright/test' );
 const { admin } = require( '../../test-data/data' );
-const { goToPostEditor } = require( '../../utils/editor' );
+const {
+	goToPostEditor,
+	fillPageTitle,
+	getCanvas,
+} = require( '../../utils/editor' );
 
 const postTitle = `Post-${ new Date().getTime().toString() }`;
 
@@ -36,13 +40,15 @@ test.describe( 'Can create a new post', () => {
 	test( 'can create new post', async ( { page } ) => {
 		await goToPostEditor( { page } );
 
-		await page
-			.getByRole( 'textbox', { name: 'Add Title' } )
-			.fill( postTitle );
+		await fillPageTitle( page, postTitle );
 
-		await page.getByRole( 'button', { name: 'Add default block' } ).click();
+		const canvas = await getCanvas( page );
 
-		await page
+		await canvas
+			.getByRole( 'button', { name: 'Add default block' } )
+			.click();
+
+		await canvas
 			.getByRole( 'document', {
 				name: 'Empty block; start writing or type forward slash to choose a block',
 			} )

plugins/woocommerce/tests/e2e-pw/tests/merchant/create-woocommerce-blocks.spec.js

@@ -1,5 +1,10 @@
 const { test, expect } = require( '@playwright/test' );
-const { disableWelcomeModal } = require( '../../utils/editor' );
+const {
+	goToPageEditor,
+	fillPageTitle,
+	insertBlock,
+	getCanvas,
+} = require( '../../utils/editor' );
 const wcApi = require( '@woocommerce/woocommerce-rest-api' ).default;
 
 const allWooBlocksPageTitle = `Insert All Woo Blocks ${ Date.now() }`;
@@ -190,46 +195,34 @@ test.describe( 'Insert All WooCommerce Blocks Into Page', () => {
 	} );
 
 	test( `can insert all WooCommerce blocks into page`, async ( { page } ) => {
-		// go to create a new page
-		await page.goto( 'wp-admin/post-new.php?post_type=page' );
+		await goToPageEditor( { page } );
 
-		await disableWelcomeModal( { page } );
+		await fillPageTitle( page, allWooBlocksPageTitle );
 
-		// fill page title
-		await page
-			.getByRole( 'textbox', { name: 'Add title' } )
-			.fill( allWooBlocksPageTitle );
-
-		// add all WC blocks and verify them as added into page
 		for ( let i = 0; i < blocks.length; i++ ) {
-			// click title field for block inserter to show up
-			await page.getByRole( 'textbox', { name: 'Add title' } ).click();
+			await test.step( `Insert ${ blocks[ i ].name } block`, async () => {
+				await insertBlock( page, blocks[ i ].name );
 
-			// add block into page
-			await page.getByLabel( 'Add block' ).click();
-			await page
-				.getByPlaceholder( 'Search', { exact: true } )
-				.fill( blocks[ i ].name );
-			await page
-				.getByRole( 'option', { name: blocks[ i ].name, exact: true } )
-				.click();
+				const canvas = await getCanvas( page );
 
-			if ( blocks[ i ].name === 'Reviews by Product' ) {
-				await page.getByLabel( simpleProductName ).check();
-				await page
-					.getByRole( 'button', { name: 'Done', exact: true } )
-					.click();
-			}
+				// eslint-disable-next-line playwright/no-conditional-in-test
+				if ( blocks[ i ].name === 'Reviews by Product' ) {
+					await canvas.getByLabel( simpleProductName ).check();
+					await canvas
+						.getByRole( 'button', { name: 'Done', exact: true } )
+						.click();
+				}
 
-			// verify added blocks into page
-			await expect(
-				page
-					.getByRole( 'document', {
-						name: `Block: ${ blocks[ i ].name }`,
-						exact: true,
-					} )
-					.first()
-			).toBeVisible();
+				// verify added blocks into page
+				await expect(
+					canvas
+						.getByRole( 'document', {
+							name: `Block: ${ blocks[ i ].name }`,
+							exact: true,
+						} )
+						.first()
+				).toBeVisible();
+			} );
 		}
 
 		// save and publish the page
@@ -246,10 +239,11 @@ test.describe( 'Insert All WooCommerce Blocks Into Page', () => {
 
 		// check all blocks inside the page after publishing
 		// except the product price due to invisibility and false-positive
+		const canvas = await getCanvas( page );
 		for ( let i = 1; i < blocks.length; i++ ) {
 			// verify added blocks into page
 			await expect(
-				page
+				canvas
 					.getByRole( 'document', {
 						name: `Block: ${ blocks[ i ].name }`,
 						exact: true,

plugins/woocommerce/tests/e2e-pw/utils/editor.js

@@ -1,3 +1,5 @@
+const { expect } = require( '@playwright/test' );
+
 const closeWelcomeModal = async ( { page } ) => {
 	// Close welcome popup if prompted
 	try {
@@ -24,6 +26,10 @@ const disableWelcomeModal = async ( { page } ) => {
 	}
 };
 
+const getCanvas = async ( page ) => {
+	return page.frame( 'editor-canvas' ) || page;
+};
+
 const goToPageEditor = async ( { page } ) => {
 	await page.goto( 'wp-admin/post-new.php?post_type=page' );
 
@@ -36,9 +42,46 @@ const goToPostEditor = async ( { page } ) => {
 	await disableWelcomeModal( { page } );
 };
 
+const fillPageTitle = async ( page, title ) => {
+	await ( await getCanvas( page ) )
+		.getByRole( 'textbox', { name: 'Add title' } )
+		.fill( title );
+};
+
+const insertBlock = async ( page, blockName ) => {
+	const canvas = await getCanvas( page );
+	// Click the title to activate the block inserter.
+	await canvas.getByRole( 'textbox', { name: 'Add title' } ).click();
+	await canvas.getByLabel( 'Add block' ).click();
+	await page.getByPlaceholder( 'Search', { exact: true } ).fill( blockName );
+	await page.getByRole( 'option', { name: blockName, exact: true } ).click();
+};
+
+const transformIntoBlocks = async ( page ) => {
+	const canvas = await getCanvas( page );
+
+	await expect(
+		canvas.locator(
+			'.wp-block-woocommerce-classic-shortcode__placeholder-copy'
+		)
+	).toBeVisible();
+	await canvas
+		.getByRole( 'button' )
+		.filter( { hasText: 'Transform into blocks' } )
+		.click();
+
+	await expect( page.getByLabel( 'Dismiss this notice' ) ).toContainText(
+		'Classic shortcode transformed to blocks.'
+	);
+};
+
 module.exports = {
 	closeWelcomeModal,
 	goToPageEditor,
 	goToPostEditor,
 	disableWelcomeModal,
+	getCanvas,
+	fillPageTitle,
+	insertBlock,
+	transformIntoBlocks,
 };