Run a full site reset on daily runs (#40061)

.github/workflows/smoke-test-daily.yml

@@ -32,6 +32,7 @@ jobs:
             CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }}
             CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }}
             DEFAULT_TIMEOUT_OVERRIDE: 120000
+            UPDATE_WC: 'nightly'
         steps:
             - uses: actions/checkout@v3
 
@@ -45,12 +46,6 @@ jobs:
               working-directory: plugins/woocommerce
               run: pnpm exec playwright install chromium
 
-            - name: Run 'Update WooCommerce' test.
-              working-directory: plugins/woocommerce
-              env:
-                  UPDATE_WC: nightly
-              run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js update-woocommerce.spec.js
-
             - name: Run API tests.
               working-directory: plugins/woocommerce
               env:

plugins/woocommerce/changelog/api-full-site-reset-on-daily

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Run a full reset on API daily test site

plugins/woocommerce/tests/api-core-tests/global-setup.js

@@ -1,8 +1,201 @@
-const { site } = require( './utils' );
+const { GITHUB_TOKEN, UPDATE_WC } = process.env;
+const { downloadZip, deleteZip } = require( './utils/plugin-utils' );
+const axios = require( 'axios' ).default;
 
 module.exports = async ( config ) => {
-	// If BASE_URL is configured, we can assume we're on CI
+	// If API_BASE_URL is configured and doesn't include localhost, running on daily host
-	if ( process.env.API_BASE_URL ) {
+	if (
-		await site.reset( process.env.USER_KEY, process.env.USER_SECRET );
+		process.env.API_BASE_URL &&
+		! process.env.API_BASE_URL.includes( 'localhost' )
+	) {
+		const { chromium, expect } = require( '@playwright/test' );
+
+		const { baseURL, userAgent } = config.projects[ 0 ].use;
+		const contextOptions = { baseURL, userAgent };
+
+		const browser = await chromium.launch();
+		const setupContext = await browser.newContext( contextOptions );
+		const setupPage = await setupContext.newPage();
+
+		const getWCDownloadURL = async () => {
+			const requestConfig = {
+				method: 'get',
+				url: 'https://api.github.com/repos/woocommerce/woocommerce/releases',
+				headers: {
+					Accept: 'application/vnd.github+json',
+				},
+				params: {
+					per_page: 100,
+				},
+			};
+			if ( GITHUB_TOKEN ) {
+				requestConfig.headers.Authorization = `Bearer ${ GITHUB_TOKEN }`;
+			}
+			const response = await axios( requestConfig ).catch( ( error ) => {
+				if ( error.response ) {
+					console.log( error.response.data );
+				}
+				throw new Error( error.message );
+			} );
+			const releaseWithTagName = response.data.find(
+				( { tag_name } ) => tag_name === UPDATE_WC
+			);
+			if ( ! releaseWithTagName ) {
+				throw new Error(
+					`No release with tag_name="${ UPDATE_WC }" found. If "${ UPDATE_WC }" is a draft release, make sure to specify a GITHUB_TOKEN environment variable.`
+				);
+			}
+			const wcZipAsset = releaseWithTagName.assets.find( ( { name } ) =>
+				name.match( /^woocommerce(-trunk-nightly)?\.zip$/ )
+			);
+			if ( wcZipAsset ) {
+				return GITHUB_TOKEN
+					? wcZipAsset.url
+					: wcZipAsset.browser_download_url;
+			}
+			throw new Error(
+				`WooCommerce release with tag "${ UPDATE_WC }" found, but does not have a WooCommerce ZIP asset.`
+			);
+		};
+
+		const url = await getWCDownloadURL();
+		const params = { url };
+
+		if ( GITHUB_TOKEN ) {
+			params.authorizationToken = GITHUB_TOKEN;
+		}
+
+		const woocommerceZipPath = await downloadZip( params );
+
+		let adminLoggedIn = false;
+		let pluginActive = false;
+
+		console.log( '--------------------------------------' );
+		console.log( 'Running daily tests, resetting site...' );
+		console.log( '--------------------------------------' );
+
+		const adminRetries = 5;
+		for ( let i = 0; i < adminRetries; i++ ) {
+			try {
+				console.log( 'Trying to log-in as admin...' );
+				await setupPage.goto( '/wp-admin' );
+				await setupPage
+					.locator( 'input[name="log"]' )
+					.fill( process.env.USER_KEY );
+				await setupPage
+					.locator( 'input[name="pwd"]' )
+					.fill( process.env.USER_SECRET );
+				await setupPage.locator( 'text=Log In' ).click();
+
+				await expect( setupPage.locator( 'div.wrap > h1' ) ).toHaveText(
+					'Dashboard'
+				);
+				console.log( 'Logged-in as admin successfully.' );
+				adminLoggedIn = true;
+				break;
+			} catch ( e ) {
+				console.log(
+					`Admin log-in failed, Retrying... ${ i }/${ adminRetries }`
+				);
+				console.log( e );
+			}
+		}
+
+		if ( ! adminLoggedIn ) {
+			console.error(
+				'Cannot proceed api test, as admin login failed. Please check if the test site has been setup correctly.'
+			);
+			process.exit( 1 );
+		}
+
+		await setupPage.goto( 'wp-admin/plugins.php' );
+		await expect( setupPage.locator( 'div.wrap > h1' ) ).toHaveText(
+			'Plugins'
+		);
+
+		console.log( 'Deactivating WooCommerce Plugin...' );
+		await setupPage.locator( '#deactivate-woocommerce' ).click();
+		await expect( setupPage.locator( 'div#message' ) ).toHaveText(
+			'Plugin deactivated.Dismiss this notice.'
+		);
+
+		console.log( 'Deleting WooCommerce Plugin...' );
+		setupPage.on( 'dialog', ( dialog ) => dialog.accept() );
+		await setupPage.locator( '#delete-woocommerce' ).click();
+		await expect( setupPage.locator( '#woocommerce-deleted' ) ).toHaveText(
+			'WooCommerce was successfully deleted.'
+		);
+
+		for ( let i = 0; i < adminRetries; i++ ) {
+			try {
+				console.log( 'Reinstalling WooCommerce Plugin...' );
+				await setupPage.goto( 'wp-admin/plugin-install.php' );
+				await setupPage.locator( 'a.upload-view-toggle' ).click();
+				await expect(
+					setupPage.locator( 'p.install-help' )
+				).toBeVisible();
+				await expect(
+					setupPage.locator( 'p.install-help' )
+				).toContainText(
+					'If you have a plugin in a .zip format, you may install or update it by uploading it here.'
+				);
+				const [ fileChooser ] = await Promise.all( [
+					setupPage.waitForEvent( 'filechooser' ),
+					setupPage.locator( '#pluginzip' ).click(),
+				] );
+				await fileChooser.setFiles( woocommerceZipPath );
+				console.log( 'Uploading nightly build...' );
+				await setupPage
+					.locator( '#install-plugin-submit' )
+					.click( { timeout: 60000 } );
+				await setupPage.waitForLoadState( 'networkidle' );
+				await expect(
+					setupPage.getByRole( 'link', { name: 'Activate Plugin' } )
+				).toBeVisible();
+				console.log( 'Activating Plugin...' );
+				await setupPage
+					.getByRole( 'link', { name: 'Activate Plugin' } )
+					.click( { timeout: 60000 } );
+				pluginActive = true;
+				break;
+			} catch ( e ) {
+				console.log(
+					`Installing and activating plugin failed, Retrying... ${ i }/${ adminRetries }`
+				);
+				console.log( e );
+			}
+		}
+		if ( ! pluginActive ) {
+			console.error(
+				'Cannot proceed api test, as installing WC failed. Please check if the test site has been setup correctly.'
+			);
+			process.exit( 1 );
+		}
+
+		console.log( 'WooCommerce Re-installed.' );
+		await expect(
+			setupPage.getByRole( 'heading', { name: 'Welcome to Woo!' } )
+		).toBeVisible();
+
+		await deleteZip( woocommerceZipPath );
+
+		// Might need to update the database
+		await setupPage.goto( 'wp-admin/plugins.php' );
+		const updateButton = setupPage.locator(
+			'text=Update WooCommerce Database'
+		);
+		const updateCompleteMessage = setupPage.locator(
+			'text=WooCommerce database update complete.'
+		);
+		await expect( setupPage.locator( 'div.wrap > h1' ) ).toHaveText(
+			'Plugins'
+		);
+		if ( await updateButton.isVisible() ) {
+			console.log( 'Database update button present. Click it.' );
+			await updateButton.click( { timeout: 60000 } );
+			await expect( updateCompleteMessage ).toBeVisible();
+		} else {
+			console.log( 'No DB update needed' );
+		}
 	}
 };

plugins/woocommerce/tests/api-core-tests/tests/data/data-crud.test.js

@@ -7787,112 +7787,56 @@ test.describe( 'Data API tests', () => {
 				} ),
 			] )
 		);
-		if ( ! shouldSkip ) {
+		expect( responseJSON ).toEqual(
-			expect( responseJSON ).toEqual(
+			expect.arrayContaining( [
-				expect.arrayContaining( [
+				expect.objectContaining( {
-					expect.objectContaining( {
+					code: 'VEF',
-						code: 'VEF',
+					name: 'Venezuelan bolívar (2008–2018)',
-						name: 'Venezuelan bolívar (2008–2018)',
+					symbol: 'Bs F',
-						symbol: 'Bs F',
+					_links: {
-						_links: {
+						self: [
-							self: [
+							{
-								{
+								href: expect.stringContaining(
-									href: expect.stringContaining(
+									'data/currencies/VEF'
-										'data/currencies/VEF'
+								),
-									),
+							},
-								},
+						],
-							],
+						collection: [
-							collection: [
+							{
-								{
+								href: expect.stringContaining(
-									href: expect.stringContaining(
+									'data/currencies'
-										'data/currencies'
+								),
-									),
+							},
-								},
+						],
-							],
+					},
-						},
+				} ),
-					} ),
+			] )
-				] )
+		);
-			);
+		expect( responseJSON ).toEqual(
-		} else {
+			expect.arrayContaining( [
-			expect( responseJSON ).toEqual(
+				expect.objectContaining( {
-				expect.arrayContaining( [
+					code: 'VES',
-					expect.objectContaining( {
+					name: 'Venezuelan bolívar',
-						code: 'VEF',
+					symbol: 'Bs.',
-						name: 'Venezuelan bolívar',
+					_links: {
-						symbol: 'Bs F',
+						self: [
-						_links: {
+							{
-							self: [
+								href: expect.stringContaining(
-								{
+									'data/currencies/VES'
-									href: expect.stringContaining(
+								),
-										'data/currencies/VEF'
+							},
-									),
+						],
-								},
+						collection: [
-							],
+							{
-							collection: [
+								href: expect.stringContaining(
-								{
+									'data/currencies'
-									href: expect.stringContaining(
+								),
-										'data/currencies'
+							},
-									),
+						],
-								},
+					},
-							],
+				} ),
-						},
+			] )
-					} ),
+		);
-				] )
-			);
-		}
-		if ( ! shouldSkip ) {
-			expect( responseJSON ).toEqual(
-				expect.arrayContaining( [
-					expect.objectContaining( {
-						code: 'VES',
-						name: 'Venezuelan bolívar',
-						symbol: 'Bs.',
-						_links: {
-							self: [
-								{
-									href: expect.stringContaining(
-										'data/currencies/VES'
-									),
-								},
-							],
-							collection: [
-								{
-									href: expect.stringContaining(
-										'data/currencies'
-									),
-								},
-							],
-						},
-					} ),
-				] )
-			);
-		} else {
-			expect( responseJSON ).toEqual(
-				expect.arrayContaining( [
-					expect.objectContaining( {
-						code: 'VES',
-						name: 'Bolívar soberano',
-						symbol: 'Bs.S',
-						_links: {
-							self: [
-								{
-									href: expect.stringContaining(
-										'data/currencies/VES'
-									),
-								},
-							],
-							collection: [
-								{
-									href: expect.stringContaining(
-										'data/currencies'
-									),
-								},
-							],
-						},
-					} ),
-				] )
-			);
-		}
 		expect( responseJSON ).toEqual(
 			expect.arrayContaining( [
 				expect.objectContaining( {

plugins/woocommerce/tests/api-core-tests/tests/shipping/shipping-zones.test.js

@@ -9,7 +9,7 @@ const shouldSkip = API_BASE_URL != undefined;
  * @group shipping
  */
 
-test.describe( 'Shipping zones API tests', () => {
+test.describe.serial( 'Shipping zones API tests', () => {
 
 	//Shipping zone to be created, retrieved, updated, and deleted by the tests.
 	let shippingZone = getShippingZoneExample();
@@ -169,9 +169,9 @@ test.describe( 'Shipping zones API tests', () => {
 		});
 
 		const putResponseStateOnlyJSON = await putResponseStateOnly.json();
-		expect( putResponseStateOnly.status() ).toEqual( 200 );
+		await expect( putResponseStateOnly.status() ).toEqual( 200 );
 
-		expect.soft( putResponseStateOnlyJSON ).toHaveLength(0);
+		await expect( putResponseStateOnlyJSON ).toHaveLength(0);
 
 	} );
 
@@ -185,15 +185,15 @@ test.describe( 'Shipping zones API tests', () => {
 		const deleteResponseJSON = await deleteResponse.json();
 
 		//validate response
-		expect( deleteResponse.status() ).toEqual( 200 );
+		await expect( deleteResponse.status() ).toEqual( 200 );
-		expect( deleteResponseJSON.id ).toEqual( shippingZone.id );
+		await expect( deleteResponseJSON.id ).toEqual( shippingZone.id );
 
 		// only run on wp-env because caching on external hosts makes unreliable
 		if ( ! shouldSkip ) {
 			//call API to attempt to retrieve the deleted shipping zone
 			const response = await request.get( `/wp-json/wc/v3/shipping/zones/${shippingZone.id}`);
 			//validate response
-			expect( response.status() ).toEqual( 404 );
+			await expect( response.status() ).toEqual( 404 );
 		}
 	} );
 

plugins/woocommerce/tests/api-core-tests/utils/plugin-utils.js

@@ -0,0 +1,260 @@
+const { APIRequest } = require( '@playwright/test' );
+const axios = require( 'axios' ).default;
+const fs = require( 'fs' );
+const path = require( 'path' );
+const { promisify } = require( 'util' );
+const execAsync = promisify( require( 'child_process' ).exec );
+
+/**
+ * Encode basic auth username and password to be used in HTTP Authorization header.
+ *
+ * @param {string} username
+ * @param {string} password
+ * @returns Base64-encoded string
+ */
+const encodeCredentials = ( username, password ) => {
+	return Buffer.from( `${ username }:${ password }` ).toString( 'base64' );
+};
+
+/**
+ * Deactivate and delete a plugin specified by the given `slug` using the WordPress API.
+ *
+ * @param {object} params
+ * @param {APIRequest} params.request
+ * @param {string} params.baseURL
+ * @param {string} params.slug
+ * @param {string} params.username
+ * @param {string} params.password
+ */
+export const deletePlugin = async ( {
+	request,
+	baseURL,
+	slug,
+	username,
+	password,
+} ) => {
+	// Check if plugin is installed by getting the list of installed plugins, and then finding the one whose `textdomain` property equals `slug`.
+	const apiContext = await request.newContext( {
+		baseURL,
+		extraHTTPHeaders: {
+			Authorization: `Basic ${ encodeCredentials( username, password ) }`,
+			cookie: '',
+		},
+	} );
+	const listPluginsResponse = await apiContext.get(
+		`/wp-json/wp/v2/plugins`,
+		{
+			failOnStatusCode: true,
+		}
+	);
+	const pluginsList = await listPluginsResponse.json();
+	const pluginToDelete = pluginsList.find(
+		( { textdomain } ) => textdomain === slug
+	);
+
+	// If installed, get its `plugin` value and use it to deactivate and delete it.
+	if ( pluginToDelete ) {
+		const { plugin } = pluginToDelete;
+		const requestURL = `/wp-json/wp/v2/plugins/${ plugin }`;
+
+		await apiContext.put( requestURL, {
+			data: { status: 'inactive' },
+		} );
+
+		await apiContext.delete( requestURL );
+	}
+};
+
+/**
+ * Download the zip file from a remote location.
+ *
+ * @param {object} param
+ * @param {string} param.url
+ * @param {string} param.repository
+ * @param {string} param.authorizationToken
+ * @param {boolean} param.prerelease
+ * @param {string} param.downloadDir
+ *
+ * @param {string} url The URL where the zip file is located. Takes precedence over `repository`.
+ * @param {string} repository The repository owner and name. For example: `woocommerce/woocommerce`. Ignored when `url` was given.
+ * @param {string} authorizationToken Authorization token used to authenticate with the GitHub API if required.
+ * @param {boolean} prerelease Flag on whether to get a prelease or not. Default `false`.
+ * @param {string} downloadDir Relative path to the download directory. Non-existing folders will be auto-created. Defaults to `tmp` under current working directory.
+ *
+ * @return {string} Absolute path to the downloaded zip.
+ */
+export const downloadZip = async ( {
+	url,
+	repository,
+	authorizationToken,
+	prerelease = false,
+	downloadDir = 'tmp',
+} ) => {
+	let zipFilename = path.basename( url || repository );
+	zipFilename = zipFilename.endsWith( '.zip' )
+		? zipFilename
+		: zipFilename.concat( '.zip' );
+	const zipFilePath = path.resolve( downloadDir, zipFilename );
+
+	let response;
+
+	// Create destination folder.
+	fs.mkdirSync( downloadDir, { recursive: true } );
+
+	const downloadURL =
+		url ??
+		( await getLatestReleaseZipUrl( {
+			repository,
+			authorizationToken,
+			prerelease,
+		} ) );
+
+	// Download the zip.
+	const options = {
+		method: 'get',
+		url: downloadURL,
+		responseType: 'stream',
+		headers: {
+			Authorization: authorizationToken
+				? `token ${ authorizationToken }`
+				: '',
+			Accept: 'application/octet-stream',
+		},
+	};
+
+	response = await axios( options ).catch( ( error ) => {
+		if ( error.response ) {
+			console.error( error.response.data );
+		}
+		throw new Error( error.message );
+	} );
+
+	response.data.pipe( fs.createWriteStream( zipFilePath ) );
+
+	return zipFilePath;
+};
+
+/**
+ * Delete a zip file. Useful when cleaning up downloaded plugin zips.
+ *
+ * @param {string} zipFilePath Local file path to the ZIP.
+ */
+export const deleteZip = async ( zipFilePath ) => {
+	await fs.unlink( zipFilePath, ( err ) => {
+		if ( err ) throw err;
+	} );
+};
+
+/**
+ * Get the download URL of the latest release zip for a plugin using GitHub API.
+ *
+ * @param {{repository: string, authorizationToken: string, prerelease: boolean, perPage: number}} param
+ * @param {string} repository The repository owner and name. For example: `woocommerce/woocommerce`.
+ * @param {string} authorizationToken Authorization token used to authenticate with the GitHub API if required.
+ * @param {boolean} prerelease Flag on whether to get a prelease or not.
+ * @param {number} perPage Limit of entries returned from the latest releases list, defaults to 3.
+ * @return {string} Download URL for the release zip file.
+ */
+export const getLatestReleaseZipUrl = async ( {
+	repository,
+	authorizationToken,
+	prerelease = false,
+	perPage = 3,
+} ) => {
+	let release;
+
+	const requesturl = prerelease
+		? `https://api.github.com/repos/${ repository }/releases?per_page=${ perPage }`
+		: `https://api.github.com/repos/${ repository }/releases/latest`;
+
+	const options = {
+		method: 'get',
+		url: requesturl,
+		headers: {
+			Authorization: authorizationToken
+				? `token ${ authorizationToken }`
+				: '',
+		},
+	};
+
+	// Get the first prerelease, or the latest release.
+	let response;
+	try {
+		response = await axios( options );
+	} catch ( error ) {
+		let errorMessage =
+			'Something went wrong when downloading the plugin.\n';
+
+		if ( error.response ) {
+			// The request was made and the server responded with a status code
+			// that falls out of the range of 2xx
+			errorMessage = errorMessage.concat(
+				`Response status: ${ error.response.status } ${ error.response.statusText }`,
+				'\n',
+				`Response body:`,
+				'\n',
+				JSON.stringify( error.response.data, null, 2 ),
+				'\n'
+			);
+		} else if ( error.request ) {
+			// The request was made but no response was received
+			// `error.request` is an instance of XMLHttpRequest in the browser and an instance of
+			// http.ClientRequest in node.js
+			errorMessage = errorMessage.concat(
+				JSON.stringify( error.request, null, 2 ),
+				'\n'
+			);
+		} else {
+			// Something happened in setting up the request that triggered an Error
+			errorMessage = errorMessage.concat( error.toJSON(), '\n' );
+		}
+
+		throw new Error( errorMessage );
+	}
+
+	release = prerelease
+		? response.data.find( ( { prerelease } ) => prerelease )
+		: response.data;
+
+	// If response contains assets, return URL of first asset.
+	// Otherwise, return the github.com URL from the tag name.
+	const { assets } = release;
+	if ( assets && assets.length ) {
+		return assets[ 0 ].url;
+	} else {
+		const tagName = release.tag_name;
+		return `https://github.com/${ repository }/archive/${ tagName }.zip`;
+	}
+};
+
+/**
+ * Install a plugin using WP CLI within a WP ENV environment.
+ * This is a workaround to the "The uploaded file exceeds the upload_max_filesize directive in php.ini" error encountered when uploading a plugin to the local WP Env E2E environment through the UI.
+ *
+ * @see https://github.com/WordPress/gutenberg/issues/29430
+ *
+ * @param {string} pluginPath
+ */
+export const installPluginThruWpCli = async ( pluginPath ) => {
+	const runWpCliCommand = async ( command ) => {
+		const { stdout, stderr } = await execAsync(
+			`pnpm exec wp-env run tests-cli -- ${ command }`
+		);
+
+		console.log( stdout );
+		console.error( stderr );
+	};
+
+	const wpEnvPluginPath = pluginPath.replace(
+		/.*\/plugins\/woocommerce/,
+		'wp-content/plugins/woocommerce'
+	);
+
+	await runWpCliCommand( `ls  ${ wpEnvPluginPath }` );
+
+	await runWpCliCommand(
+		`wp plugin install --activate --force ${ wpEnvPluginPath }`
+	);
+
+	await runWpCliCommand( `wp plugin list` );
+};