Performance: Bootstrap metrics tracking

.gitignore

@@ -104,3 +104,7 @@ changes.json
 
 # default docs manifest
 /manifest.json
+
+# Metrics tests
+/artifacts
+/plugins/*/artifacts

.syncpackrc

@@ -171,6 +171,15 @@
 			],
 			"pinVersion": "^8.13.0"
 		},
+		{
+			"dependencies": [
+				"@wordpress/e2e-test-utils-playwright"
+			],
+			"packages": [
+				"**"
+			],
+			"pinVersion": "wp-6.4"
+		},
 		{
 			"dependencies": [
 				"@wordpress/**"

plugins/woocommerce/changelog/add-metrics-tests

@@ -0,0 +1,5 @@
+Significance: patch
+Type: dev
+Comment: It's a build process E2E change.
+
+

plugins/woocommerce/package.json

@@ -49,6 +49,7 @@
 		"test:e2e-pw": "USE_WP_ENV=1 pnpm playwright test --config=tests/e2e-pw/playwright.config.js",
 		"test:env:start": "pnpm env:test",
 		"test:php": "./vendor/bin/phpunit -c ./phpunit.xml",
+		"test:metrics": "USE_WP_ENV=1 pnpm playwright test --config=tests/metrics/playwright.config.js",
 		"test:php:env": "wp-env run --env-cwd='wp-content/plugins/woocommerce' tests-cli vendor/bin/phpunit -c phpunit.xml --verbose",
 		"test:unit": "pnpm test:php",
 		"test:unit:env": "pnpm test:php:env",
@@ -137,6 +138,7 @@
 		"@woocommerce/woocommerce-rest-api": "^1.0.1",
 		"@wordpress/babel-plugin-import-jsx-pragma": "1.1.3",
 		"@wordpress/babel-preset-default": "3.0.2",
+		"@wordpress/e2e-test-utils-playwright": "wp-6.4",
 		"@wordpress/env": "^8.13.0",
 		"@wordpress/stylelint-config": "19.1.0",
 		"allure-commandline": "^2.25.0",
@@ -220,9 +222,9 @@
 				"node_modules/@woocommerce/e2e-core-tests/CHANGELOG.md",
 				"node_modules/@woocommerce/api/dist/",
 				"node_modules/@woocommerce/admin-e2e-tests/build",
+				"node_modules/@woocommerce/classic-assets/build",
 				"node_modules/@woocommerce/block-library/build",
 				"node_modules/@woocommerce/block-library/blocks.ini",
-				"node_modules/@woocommerce/classic-assets/build",
 				"node_modules/@woocommerce/admin-library/build",
 				"package.json",
 				"!node_modules/@woocommerce/admin-e2e-tests/*.ts.map",

plugins/woocommerce/tests/metrics/config/global-setup.ts

@@ -0,0 +1,23 @@
+import { request } from '@playwright/test';
+import type { FullConfig } from '@playwright/test';
+import { RequestUtils } from '@wordpress/e2e-test-utils-playwright';
+
+async function globalSetup( config: FullConfig ) {
+	const { storageState, baseURL } = config.projects[ 0 ].use;
+	const storageStatePath =
+		typeof storageState === 'string' ? storageState : undefined;
+
+	const requestContext = await request.newContext( {
+		baseURL,
+	} );
+
+	const requestUtils = new RequestUtils( requestContext, {
+		storageStatePath,
+	} );
+
+	// Authenticate and save the storageState to disk.
+	await requestUtils.setupRest();
+	await requestContext.dispose();
+}
+
+export default globalSetup;

plugins/woocommerce/tests/metrics/config/performance-reporter.ts

@@ -0,0 +1,73 @@
+import path from 'path';
+import { writeFileSync } from 'fs';
+import type {
+	Reporter,
+	FullResult,
+	TestCase,
+	TestResult,
+} from '@playwright/test/reporter';
+
+export type WPPerformanceResults = Record< string, number >;
+
+class PerformanceReporter implements Reporter {
+	private results: Record< string, WPPerformanceResults >;
+
+	constructor() {
+		this.results = {};
+	}
+
+	onTestEnd( test: TestCase, result: TestResult ): void {
+		for ( const attachment of result.attachments ) {
+			if ( attachment.name !== 'results' ) {
+				continue;
+			}
+
+			if ( ! attachment.body ) {
+				throw new Error( 'Empty results attachment' );
+			}
+
+			const testSuite = path.basename( test.location.file, '.spec.js' );
+			const resultsId = process.env.RESULTS_ID || testSuite;
+			const resultsPath = process.env.WP_ARTIFACTS_PATH as string;
+			const resultsBody = attachment.body.toString();
+			const results = JSON.parse( resultsBody );
+
+			// Save curated results to file.
+			writeFileSync(
+				path.join(
+					resultsPath,
+					`${ resultsId }.performance-results.json`
+				),
+				JSON.stringify( results, null, 2 )
+			);
+
+			this.results[ testSuite ] = results;
+		}
+	}
+
+	onEnd( result: FullResult ) {
+		if ( result.status !== 'passed' ) {
+			return;
+		}
+
+		if ( process.env.CI ) {
+			return;
+		}
+
+		// Print the results.
+		for ( const [ testSuite, results ] of Object.entries( this.results ) ) {
+			const printableResults: Record< string, { value: string } > = {};
+
+			for ( const [ key, value ] of Object.entries( results ) ) {
+				printableResults[ key ] = { value: `${ value } ms` };
+			}
+
+			// eslint-disable-next-line no-console
+			console.log( `\n${ testSuite }\n` );
+			// eslint-disable-next-line no-console
+			console.table( printableResults );
+		}
+	}
+}
+
+export default PerformanceReporter;

plugins/woocommerce/tests/metrics/fixtures/index.ts

@@ -0,0 +1,110 @@
+import fs from 'fs';
+import path from 'path';
+import type { Page } from '@playwright/test';
+import { readFile } from '../utils.js';
+import { expect } from '@wordpress/e2e-test-utils-playwright';
+
+type PerfUtilsConstructorProps = {
+	page: Page;
+};
+
+export class PerfUtils {
+	page: Page;
+
+	constructor( { page }: PerfUtilsConstructorProps ) {
+		this.page = page;
+	}
+
+	/**
+	 * Returns the locator for the editor canvas element. This supports both the
+	 * legacy and the iframed canvas.
+	 *
+	 * @return Locator for the editor canvas element.
+	 */
+	async getCanvas() {
+		const canvasLocator = this.page.locator(
+			'.wp-block-post-content, iframe[name=editor-canvas]'
+		);
+
+		const isFramed = await canvasLocator.evaluate(
+			( node ) => node.tagName === 'IFRAME'
+		);
+
+		if ( isFramed ) {
+			return canvasLocator.frameLocator( ':scope' );
+		}
+
+		return canvasLocator;
+	}
+
+	/**
+	 * Saves the post as a draft and returns its URL.
+	 *
+	 * @return URL of the saved draft.
+	 */
+	async saveDraft() {
+		await this.page.getByRole( 'button', { name: 'Save draft' } ).click();
+		await expect(
+			this.page.getByRole( 'button', { name: 'Saved' } )
+		).toBeDisabled();
+
+		const postId = new URL( this.page.url() ).searchParams.get( 'post' );
+
+		return postId;
+	}
+
+	/**
+	 * Disables the editor autosave function.
+	 */
+	async disableAutosave() {
+		await this.page.waitForFunction( () => window?.wp?.data );
+
+		await this.page.evaluate( () => {
+			return window.wp.data
+				.dispatch( 'core/editor' )
+				.updateEditorSettings( {
+					autosaveInterval: 100000000000,
+					localAutosaveInterval: 100000000000,
+				} );
+		} );
+	}
+
+	/**
+	 * Loads blocks from the large post fixture into the editor canvas.
+	 */
+	async loadBlocksForLargePost() {
+		return await this.loadBlocksFromHtml(
+			path.join( process.env.ASSETS_PATH!, 'large-post.html' )
+		);
+	}
+
+	/**
+	 * Loads blocks from an HTML fixture with given path into the editor canvas.
+	 *
+	 * @param filepath Path to the HTML fixture.
+	 */
+	async loadBlocksFromHtml( filepath: string ) {
+		if ( ! fs.existsSync( filepath ) ) {
+			throw new Error( `File not found: ${ filepath }` );
+		}
+
+		await this.page.waitForFunction(
+			() => window?.wp?.blocks && window?.wp?.data
+		);
+
+		return await this.page.evaluate( ( html: string ) => {
+			const { parse } = window.wp.blocks;
+			const { dispatch } = window.wp.data;
+			const blocks = parse( html );
+
+			blocks.forEach( ( block: any ) => {
+				if ( block.name === 'core/image' ) {
+					delete block.attributes.id;
+					delete block.attributes.url;
+				}
+			} );
+
+			dispatch( 'core/block-editor' ).resetBlocks( blocks );
+		}, readFile( filepath ) );
+	}
+}

plugins/woocommerce/tests/metrics/playwright.config.js

@@ -0,0 +1,58 @@
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { defineConfig, devices } from '@playwright/test';
+
+process.env.ASSETS_PATH = path.join( __dirname, 'assets' );
+process.env.WP_ARTIFACTS_PATH ??= path.join( process.cwd(), 'artifacts' );
+process.env.STORAGE_STATE_PATH ??= path.join(
+	process.env.WP_ARTIFACTS_PATH,
+	'storage-states/admin.json'
+);
+process.env.WP_BASE_URL ??= 'http://localhost:8086';
+
+const config = defineConfig( {
+	reporter: process.env.CI
+		? './config/performance-reporter.ts'
+		: [ [ 'list' ], [ './config/performance-reporter.ts' ] ],
+	forbidOnly: !! process.env.CI,
+	fullyParallel: false,
+	workers: 1,
+	retries: 0,
+	timeout: parseInt( process.env.TIMEOUT || '', 10 ) || 600_000, // Defaults to 10 minutes.
+	// Don't report slow test "files", as we will be running our tests in serial.
+	reportSlowTests: null,
+	testDir: './specs',
+	outputDir: path.join( process.env.WP_ARTIFACTS_PATH, 'test-results' ),
+	snapshotPathTemplate:
+		'{testDir}/{testFileDir}/__snapshots__/{arg}-{projectName}{ext}',  
+	globalSetup: fileURLToPath(
+		new URL( './config/global-setup.ts', 'file:' + __filename ).href
+	),
+	use: {
+		baseURL: process.env.WP_BASE_URL || 'http://localhost:8086',
+		headless: true,
+		viewport: {
+			width: 960,
+			height: 700,
+		},
+		ignoreHTTPSErrors: true,
+		locale: 'en-US',
+		contextOptions: {
+			reducedMotion: 'reduce',
+			strictSelectors: true,
+		},
+		storageState: process.env.STORAGE_STATE_PATH,
+		actionTimeout: 120_000, // 2 minutes.
+		trace: 'retain-on-failure',
+		screenshot: 'only-on-failure',
+		video: 'off',
+	},
+	projects: [
+		{
+			name: 'chromium',
+			use: { ...devices[ 'Desktop Chrome' ] },
+		},
+	],
+} );
+
+export default config;

plugins/woocommerce/tests/metrics/specs/editor.spec.js

@@ -0,0 +1,178 @@
+/* eslint-disable playwright/no-conditional-in-test, playwright/expect-expect */
+
+/**
+ * WordPress dependencies
+ */
+import { test, Metrics } from '@wordpress/e2e-test-utils-playwright';
+
+/**
+ * Internal dependencies
+ */
+import { PerfUtils } from '../fixtures';
+import { median } from '../utils';
+
+// See https://github.com/WordPress/gutenberg/issues/51383#issuecomment-1613460429
+const BROWSER_IDLE_WAIT = 1000;
+
+const results = {};
+
+async function editPost( admin, page, postId ) {
+	const query = new URLSearchParams();
+	query.set( 'post', String( postId ) );
+	query.set( 'action', 'edit' );
+
+	await admin.visitAdminPage( 'post.php', query.toString() );
+	await setPreferences( page, 'core/edit-post', {
+		welcomeGuide: false,
+		fullscreenMode: false,
+	} );
+}
+
+async function setPreferences( page, context, preferences ) {
+	await page.waitForFunction( () => window?.wp?.data );
+
+	await page.evaluate(
+		async ( props ) => {
+			for ( const [ key, value ] of Object.entries(
+				props.preferences
+			) ) {
+				await window.wp.data
+					.dispatch( 'core/preferences' )
+					.set( props.context, key, value );
+			}
+		},
+		{ context, preferences }
+	);
+}
+
+test.describe( 'Editor Performance', () => {
+	test.use( {
+		perfUtils: async ( { page }, use ) => {
+			await use( new PerfUtils( { page } ) );
+		},
+		metrics: async ( { page }, use ) => {
+			await use( new Metrics( { page } ) );
+		},
+	} );
+
+	test.afterAll( async ( {}, testInfo ) => {
+		const medians = {};
+		Object.keys( results ).map( ( metric ) => {
+			medians[ metric ] = median( results[ metric ] );
+		} );
+		await testInfo.attach( 'results', {
+			body: JSON.stringify( medians, null, 2 ),
+			contentType: 'application/json',
+		} );
+	} );
+
+	test.describe( 'Loading', () => {
+		let draftId = null;
+
+		test( 'Setup the test post', async ( { admin, perfUtils } ) => {
+			await admin.createNewPost();
+			await perfUtils.loadBlocksForLargePost();
+			draftId = await perfUtils.saveDraft();
+		} );
+
+		const samples = 2;
+		const throwaway = 1;
+		const iterations = samples + throwaway;
+		for ( let i = 1; i <= iterations; i++ ) {
+			test( `Run the test (${ i } of ${ iterations })`, async ( {
+				admin,
+				page,
+				perfUtils,
+				metrics,
+			} ) => {
+				// Open the test draft.
+				await editPost( admin, page, draftId );
+				const canvas = await perfUtils.getCanvas();
+
+				// Wait for the first block.
+				await canvas.locator( '.wp-block' ).first().waitFor();
+
+				// Get the durations.
+				const loadingDurations = await metrics.getLoadingDurations();
+
+				// Save the results.
+				if ( i > throwaway ) {
+					Object.entries( loadingDurations ).forEach(
+						( [ metric, duration ] ) => {
+							const metricKey =
+								metric === 'timeSinceResponseEnd'
+									? 'firstBlock'
+									: metric;
+							if ( ! results[ metricKey ] ) {
+								results[ metricKey ] = [];
+							}
+							results[ metricKey ].push( duration );
+						}
+					);
+				}
+			} );
+		}
+	} );
+
+	test.describe( 'Typing', () => {
+		let draftId = null;
+
+		test( 'Setup the test post', async ( { admin, perfUtils, editor } ) => {
+			await admin.createNewPost();
+			await perfUtils.loadBlocksForLargePost();
+			await editor.insertBlock( { name: 'core/paragraph' } );
+			draftId = await perfUtils.saveDraft();
+			console.log( draftId );
+		} );
+
+		test( 'Run the test', async ( {
+			admin,
+			page,
+			perfUtils,
+			metrics,
+		} ) => {
+			await editPost( admin, page, draftId );
+			await perfUtils.disableAutosave();
+			const canvas = await perfUtils.getCanvas();
+
+			const paragraph = canvas.getByRole( 'document', {
+				name: /Empty block/i,
+			} );
+
+			// The first character typed triggers a longer time (isTyping change).
+			// It can impact the stability of the metric, so we exclude it. It
+			// probably deserves a dedicated metric itself, though.
+			const samples = 10;
+			const throwaway = 1;
+			const iterations = samples + throwaway;
+
+			// Start tracing.
+			await metrics.startTracing();
+
+			// Type the testing sequence into the empty paragraph.
+			await paragraph.type( 'x'.repeat( iterations ), {
+				delay: BROWSER_IDLE_WAIT,
+				// The extended timeout is needed because the typing is very slow
+				// and the `delay` value itself does not extend it.
+				timeout: iterations * BROWSER_IDLE_WAIT * 2, // 2x the total time to be safe.
+			} );
+
+			// Stop tracing.
+			await metrics.stopTracing();
+
+			// Get the durations.
+			const [ keyDownEvents, keyPressEvents, keyUpEvents ] =
+				metrics.getTypingEventDurations();
+
+			// Save the results.
+			results.type = [];
+			for ( let i = throwaway; i < iterations; i++ ) {
+				results.type.push(
+					keyDownEvents[ i ] + keyPressEvents[ i ] + keyUpEvents[ i ]
+				);
+			}
+		} );
+	} );
+} );
+
+/* eslint-enable playwright/no-conditional-in-test, playwright/expect-expect */

plugins/woocommerce/tests/metrics/utils.js

@@ -0,0 +1,21 @@
+import { existsSync, readFileSync } from 'fs';
+
+export function median( array ) {
+	if ( ! array || ! array.length ) return undefined;
+
+	const numbers = [ ...array ].sort( ( a, b ) => a - b );
+	const middleIndex = Math.floor( numbers.length / 2 );
+
+	if ( numbers.length % 2 === 0 ) {
+		return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
+	}
+	return numbers[ middleIndex ];
+}
+
+export function readFile( filePath ) {
+	if ( ! existsSync( filePath ) ) {
+		throw new Error( `File does not exist: ${ filePath }` );
+	}
+
+	return readFileSync( filePath, 'utf8' ).trim();
+}