Merge pull request #30863 from woocommerce/add/wp-l-minus-2-testing

Support for L-2 WordPress version testing

.github/workflows/smoke-test-release.yml

@@ -49,3 +49,53 @@ jobs:
           npx wc-e2e test:e2e ./tests/e2e/specs/smoke-tests/update-woocommerce.js
           npx wc-e2e test:e2e
           npx wc-api-tests test api
+  test-wp-version:
+    name: Smoke test on L-${{ matrix.wp }} WordPress version
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix: 
+        wp: [ '1', '2' ]
+    steps:
+
+      - name: Create dirs.
+        run: |
+              mkdir -p code/woocommerce
+              mkdir -p package/woocommerce
+              mkdir -p tmp/woocommerce
+              mkdir -p node_modules
+
+      - name: Checkout code.
+        uses: actions/checkout@v2
+        with:
+          path: package/woocommerce
+
+      - name: Run npm install.
+        working-directory: package/woocommerce
+        run: npm install
+
+      - name: Load docker images and start containers.
+        working-directory: package/woocommerce
+        env:
+          LATEST_WP_VERSION_MINUS: ${{ matrix.wp }}
+        run: npx wc-e2e docker:up
+
+      - name: Move current directory to code. We will install zip file in this dir later.
+        run: mv ./package/woocommerce/* ./code/woocommerce
+
+      - name: Download WooCommerce release zip
+        working-directory: tmp
+        run: |
+          ASSET_ID=$(jq ".release.assets[0].id" $GITHUB_EVENT_PATH)
+
+          curl https://api.github.com/repos/woocommerce/woocommerce/releases/assets/${ASSET_ID} -LJOH 'Accept: application/octet-stream'
+
+          unzip woocommerce.zip -d woocommerce
+          mv woocommerce/* ../package/woocommerce/
+
+      - name: Run tests command.
+        working-directory: code/woocommerce
+        env:
+          WC_E2E_SCREENSHOTS: 1
+          E2E_SLACK_TOKEN: ${{ secrets.SMOKE_TEST_SLACK_TOKEN }}
+          E2E_SLACK_CHANNEL: ${{ secrets.RELEASE_TEST_SLACK_CHANNEL }}
+        run: npx wc-e2e test:e2e

tests/e2e/env/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Unreleased
 
+## Added
+
+- Added `LATEST_WP_VERSION_MINUS` that allows setting a number to subtract from the current WordPress version for the WordPress Docker image.
+
 # 0.2.3
 
 ## Added

tests/e2e/env/README.md

@@ -102,6 +102,18 @@ This value will override the default Jest timeout as well as pass the timeout to
 
 For a list of the methods that the above timeout affects, please see the Puppeteer documentation for [`page.setDefaultTimeout()`](https://pptr.dev/#?product=Puppeteer&version=v10.2.0&show=api-pagesetdefaulttimeouttimeout) and [`page.setDefaultNavigationTimeout`](https://pptr.dev/#?product=Puppeteer&version=v10.2.0&show=api-pagesetdefaultnavigationtimeouttimeout) for more information.
 
+### Test Against Previous WordPress Versions
+
+You can use the `LATEST_WP_VERSION_MINUS` flag to determine how many versions back from the current WordPress version to use in the Docker environment. This is calculated from the current WordPress version minus the set value. For example, if `LATEST_WP_VERSION_MINUS` is set to 1, it will calculate the current WordPress version minus one, and use that for the WordPress Docker container. 
+
+For example, you could run the following command:
+
+```bash
+LATEST_WP_VERSION_MINUS=2 npx wc-e2e docker:up
+```
+
+In this example, if the current WordPress version is 6.0, this will go two versions back and use the WordPress 5.8 Docker image for the tests.
+
 ### Jest Puppeteer Config
 
 The test sequencer uses the following default Puppeteer configuration:

tests/e2e/env/bin/docker-compose.sh

@@ -13,6 +13,10 @@ if [[ $1 ]]; then
 		export WORDPRESS_VERSION="5.8.0"
 	fi
 
+	if [[ $LATEST_WP_VERSION_MINUS ]]; then
+		export WORDPRESS_VERSION=$(./bin/get-previous-version.js $WORDPRESS_VERSION $LATEST_WP_VERSION_MINUS 2> /dev/null)
+	fi
+
 	if ! [[ $TRAVIS_PHP_VERSION =~ ^[0-9]+\.[0-9]+ ]]; then
 		TRAVIS_PHP_VERSION=$(./bin/get-latest-docker-tag.js php 7 2> /dev/null)
 	fi

tests/e2e/env/bin/get-latest-docker-tag.js

@@ -2,6 +2,7 @@
 
 const https = require( 'https' );
 const semver = require( 'semver' );
+const getLatestMinusVersion = require( './get-previous-version' );
 
 /**
  * Fetches the latest tag from a page using the Docker HTTP api.
@@ -107,6 +108,10 @@ function findLatestVersion( image, nameSearch ) {
 			return fetchLatestTagFromPage( image, nameSearch, ++page ).then( paginationFn );
 		}
 
+		if ( image === 'wordpress' && process.env.LATEST_WP_VERSION_MINUS ) {
+			return getLatestMinusVersion( latestVersion.toString(), process.env.LATEST_WP_VERSION_MINUS );
+		}
+
 		return latestVersion.toString();
 	};
 

tests/e2e/env/bin/get-previous-version.js

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+
+/**
+ *
+ * @param {latestVersion} latestVersion
+ * @param {minus} minus
+ * @returns {String} the minused version.
+ */
+function getLatestMinusVersion( latestVersion, minus ) {
+	// Convert the 1 or 2 to a decimal we can use for the logic below.
+	let minusAmount = minus / 10;
+
+	// Check if we only have a major / minor (e.g. x.x) to append a patch version
+	if ( latestVersion.match( /\./g ).length < 2 ) {
+		latestVersion = latestVersion.concat( '.0' )
+	}
+
+	const baseVersion = latestVersion.replace( /.[^\.]$/, '' );
+
+	// Calculate the version we need and return.
+	console.info( String( baseVersion - minusAmount ) );
+	process.exit( 0 );
+}
+
+const latestVersion = process.argv[2];
+const minus = process.argv[3];
+if ( ! latestVersion || ! minus ) {
+	console.error( 'Usage: get-previous-version.js <latestVersion> <minus>' );
+	process.exit( 1 );
+}
+
+getLatestMinusVersion( latestVersion, minus );