Steve-Dee-Designs
/
woocommerce
mirror of https://github.com/woocommerce/woocommerce.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[testing workflows] Add test environments and deprecate the daily e2e workflow (#48101) 

* Deprecate pr-smoke-test.yml

* Add configs for each environment

* Add scripts

* Add arguments to playwright command

* Update pnpm scripts and existing e2e projects

* Fix shellcheck warning

* Update test env start for Gutenberg stable run

* Add changelog

* Remove unused var

* Set the USE_WP_ENV var

* Configure daily e2e jobs

* Rename some steps
This commit is contained in:
Adrian Moldovan 2024-06-04 00:12:45 +03:00 committed by GitHub
parent 2a912b50ae
commit 9b251ce318
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 241 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/ci.yml vendored
View File

@ -16,6 +16,9 @@ on:
concurrency:
group: '${{ github.workflow }}-${{ github.ref }}'
cancel-in-progress: true
env:
FORCE_COLOR: 1
jobs:
project-jobs:
@ -101,7 +104,7 @@ jobs:
with:
install: '${{ matrix.projectName }}...'
build: '${{ matrix.projectName }}'
- name: 'Prepare Test Environment'
- name: 'Start Test Environment'
id: 'prepare-test-environment'
if: ${{ matrix.testEnv.shouldCreate }}
env: ${{ matrix.testEnv.envVars }}
@ -146,7 +149,7 @@ jobs:
echo "COMMIT_MESSAGE=$COMMIT_MESSAGE" >> "$GITHUB_OUTPUT"
shell: bash
- name: 'Prepare Test Environment'
- name: 'Start Test Environment'
id: 'prepare-test-environment'
if: ${{ matrix.testEnv.shouldCreate }}
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
@ -187,7 +190,7 @@ jobs:
install: '${{ matrix.projectName }}...'
build: '${{ matrix.projectName }}'
- name: 'Prepare Test Environment'
- name: 'Start Test Environment'
id: 'prepare-test-environment'
if: ${{ matrix.testEnv.shouldCreate }}
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
@ -227,7 +230,7 @@ jobs:
install: '${{ matrix.projectName }}...'
build: '${{ matrix.projectName }}'
- name: 'Prepare Test Environment'
- name: 'Start Test Environment'
id: 'prepare-test-environment'
if: ${{ matrix.testEnv.shouldCreate }}
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'

0
.github/workflows/smoke-test-daily.yml → .github/workflows/deprecated/smoke-test-daily.yml vendored
View File

4
plugins/woocommerce/changelog/e2e-environments-setup-migrate-daily-runs Normal file
View File

@ -0,0 +1,4 @@
Significance: patch
Type: dev
Added test enviornments

73
plugins/woocommerce/package.json
View File

@ -52,8 +52,9 @@
"test": "pnpm test:unit",
"test:api": "API_TEST_REPORT_DIR=\"$PWD/tests/api\" pnpm exec wc-api-tests test api",
"test:api-pw": "USE_WP_ENV=1 pnpm playwright test --config=tests/api-core-tests/playwright.config.js",
"test:e2e-pw": "USE_WP_ENV=1 pnpm playwright test --config=tests/e2e-pw/playwright.config.js --project=default",
"test:e2e:gutenberg": "pnpm playwright test --config=tests/e2e-pw/playwright.config.js --project=Gutenberg",
"test:e2e-pw": "pnpm playwright test --config=tests/e2e-pw/envs/default/playwright.config.js",
"test:e2e": "pnpm test:e2e:with-env default",
"test:e2e:with-env": "bash ./tests/e2e-pw/run-tests-with-env.sh",
"test:perf": "./k6 run ./tests/performance/tests/gh-action-pr-requests.js",
"test:env:start": "pnpm env:test",
"test:php": "./vendor/bin/phpunit -c ./phpunit.xml",
@ -225,7 +226,7 @@
{
"name": "Core e2e tests",
"testType": "e2e",
"command": "test:e2e-pw",
"command": "test:e2e:with-env default",
"shardingArguments": [
"--shard=1/5",
"--shard=2/5",
@ -251,9 +252,9 @@
]
},
{
"name": "Core e2e tests - Gutenberg",
"name": "Core e2e tests - Gutenberg stable",
"testType": "e2e",
"command": "test:e2e:gutenberg",
"command": "test:e2e:with-env gutenberg-stable",
"shardingArguments": [
"--shard=1/5",
"--shard=2/5",
@ -266,13 +267,71 @@
"daily-e2e"
],
"testEnv": {
"start": "env:test && cd plugins/woocommerce && PLUGIN_REPOSITORY='WordPress/gutenberg' PLUGIN_NAME=Gutenberg PLUGIN_SLUG=gutenberg ./tests/e2e-pw/bin/install-plugin.sh"
"start": "env:test"
}
},
{
"name": "Core e2e tests - Gutenberg nightly",
"testType": "e2e",
"command": "test:e2e:with-env gutenberg-nightly",
"shardingArguments": [
"--shard=1/5",
"--shard=2/5",
"--shard=3/5",
"--shard=4/5",
"--shard=5/5"
],
"changes": [],
"events": [
"daily-e2e"
],
"testEnv": {
"start": "env:test"
}
},
{
"name": "Core e2e tests - Woocommerce Payments",
"testType": "e2e",
"command": "test:e2e:with-env woocommerce-payments",
"shardingArguments": [],
"changes": [],
"events": [
"daily-e2e"
],
"testEnv": {
"start": "env:test"
}
},
{
"name": "Core e2e tests - Woocommerce Paypal Payments",
"testType": "e2e",
"command": "test:e2e:with-env woocommerce-paypal-payments",
"shardingArguments": [],
"changes": [],
"events": [
"daily-e2e"
],
"testEnv": {
"start": "env:test"
}
},
{
"name": "Core e2e tests - WooCommerce Shipping & Tax",
"testType": "e2e",
"command": "test:e2e:with-env woocommerce-services",
"shardingArguments": [],
"changes": [],
"events": [
"daily-e2e"
],
"testEnv": {
"start": "env:test"
}
},
{
"name": "Core e2e tests - HPOS disabled",
"testType": "e2e",
"command": "test:e2e-pw",
"command": "test:e2e:with-env default",
"shardingArguments": [
"--shard=1/5",
"--shard=2/5",

5
plugins/woocommerce/tests/e2e-pw/envs/default/env-setup.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
set -eo pipefail
echo "Default environment setup."

16
plugins/woocommerce/tests/e2e-pw/envs/default/playwright.config.js Normal file
View File

@ -0,0 +1,16 @@
let config = require( '../../playwright.config.js' );
const { devices } = require( '@playwright/test' );
process.env.USE_WP_ENV = 'true';
config = {
...config,
projects: [
{
name: 'default',
use: { ...devices[ 'Desktop Chrome' ] },
},
],
};
module.exports = config;

10
plugins/woocommerce/tests/e2e-pw/envs/gutenberg-nightly/env-setup.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
set -eo pipefail
SCRIPT_PATH=$(
cd "$(dirname "${BASH_SOURCE[0]}")" || return
pwd -P
)
PLUGIN_REPOSITORY='bph/gutenberg' PLUGIN_NAME=Gutenberg PLUGIN_SLUG=gutenberg "$SCRIPT_PATH"/../../bin/install-plugin.sh

3
plugins/woocommerce/tests/e2e-pw/envs/gutenberg-nightly/playwright.config.js Normal file
View File

@ -0,0 +1,3 @@
const config = require( '../gutenberg-stable/playwright.config.js' );
module.exports = config;

10
plugins/woocommerce/tests/e2e-pw/envs/gutenberg-stable/env-setup.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
set -eo pipefail
SCRIPT_PATH=$(
cd "$(dirname "${BASH_SOURCE[0]}")" || return
pwd -P
)
PLUGIN_REPOSITORY='WordPress/gutenberg' PLUGIN_NAME=Gutenberg PLUGIN_SLUG=gutenberg "$SCRIPT_PATH"/../../bin/install-plugin.sh

16
plugins/woocommerce/tests/e2e-pw/envs/gutenberg-stable/playwright.config.js Normal file
View File

@ -0,0 +1,16 @@
let config = require( '../../playwright.config.js' );
process.env.USE_WP_ENV = 'true';
config = {
...config,
projects: [
{
name: 'Gutenberg',
testIgnore:
/.*smoke-tests\/*|.*js-file-monitor\/*|.*admin-tasks\/*|.*activate-and-setup\/*|.*admin-analytics\/*|.*admin-marketing\/*|.*basic\/*|.*account-\/*|.*settings-\/*|.*users-\/*|.*order\/*|.*page-loads\/*/,
},
],
};
module.exports = config;

10
plugins/woocommerce/tests/e2e-pw/envs/woocommerce-payments/env-setup.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
set -eo pipefail
SCRIPT_PATH=$(
cd "$(dirname "${BASH_SOURCE[0]}")" || return
pwd -P
)
PLUGIN_REPOSITORY='automattic/woocommerce-payments' PLUGIN_NAME=WooPayments PLUGIN_SLUG=woocommerce-payments "$SCRIPT_PATH"/../../bin/install-plugin.sh

15
plugins/woocommerce/tests/e2e-pw/envs/woocommerce-payments/playwright.config.js Normal file
View File

@ -0,0 +1,15 @@
let config = require( '../../playwright.config.js' );
process.env.USE_WP_ENV = 'true';
config = {
...config,
projects: [
{
name: 'WooPayments',
testMatch: /.*basic\/*/,
},
],
};
module.exports = config;

10
plugins/woocommerce/tests/e2e-pw/envs/woocommerce-paypal-payments/env-setup.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
set -eo pipefail
SCRIPT_PATH=$(
cd "$(dirname "${BASH_SOURCE[0]}")" || return
pwd -P
)
PLUGIN_REPOSITORY='woocommerce/woocommerce-paypal-payments' PLUGIN_NAME='WooCommerce PayPal Payments' PLUGIN_SLUG=woocommerce-paypal-payments "$SCRIPT_PATH"/../../bin/install-plugin.sh

15
plugins/woocommerce/tests/e2e-pw/envs/woocommerce-paypal-payments/playwright.config.js Normal file
View File

@ -0,0 +1,15 @@
let config = require( '../../playwright.config.js' );
process.env.USE_WP_ENV = 'true';
config = {
...config,
projects: [
{
name: 'WooCommerce PayPal Payments',
testMatch: /.*basic\/*/,
},
],
};
module.exports = config;

10
plugins/woocommerce/tests/e2e-pw/envs/woocommerce-services/env-setup.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
set -eo pipefail
SCRIPT_PATH=$(
cd "$(dirname "${BASH_SOURCE[0]}")" || return
pwd -P
)
PLUGIN_REPOSITORY='automattic/woocommerce-services' PLUGIN_NAME='WooCommerce Shipping & Tax' PLUGIN_SLUG='woocommerce-services' "$SCRIPT_PATH"/../../bin/install-plugin.sh

15
plugins/woocommerce/tests/e2e-pw/envs/woocommerce-services/playwright.config.js Normal file
View File

@ -0,0 +1,15 @@
let config = require( '../../playwright.config.js' );
process.env.USE_WP_ENV = 'true';
config = {
...config,
projects: [
{
name: 'WooCommerce Shipping & Tax',
testMatch: /.*basic\/*/,
},
],
};
module.exports = config;

30
plugins/woocommerce/tests/e2e-pw/playwright.config.js
View File

@ -1,5 +1,5 @@
const { devices } = require( '@playwright/test' );
require( 'dotenv' ).config( { path: __dirname + '/.env' } );
const testsRootPath = __dirname;
const {
ALLURE_RESULTS_DIR,
@ -18,14 +18,16 @@ const reporter = [
{
outputFolder:
ALLURE_RESULTS_DIR ??
'./tests/e2e-pw/test-results/allure-results',
`${ testsRootPath }/tests/e2e-pw/test-results/allure-results`,
detail: true,
suiteTitle: true,
},
],
[
'json',
{ outputFile: `./test-results/test-results-${ Date.now() }.json` },
{
outputFile: `${ testsRootPath }/test-results/test-results-${ Date.now() }.json`,
},
],
];
@ -37,7 +39,8 @@ if ( process.env.CI ) {
'html',
{
outputFolder:
PLAYWRIGHT_HTML_REPORT ?? './test-results/playwright-report',
PLAYWRIGHT_HTML_REPORT ??
`${ testsRootPath }/test-results/playwright-report`,
open: 'on-failure',
},
] );
@ -48,10 +51,10 @@ const config = {
? Number( DEFAULT_TIMEOUT_OVERRIDE )
: 120 * 1000,
expect: { timeout: 20 * 1000 },
outputDir: './test-results/results-data',
outputDir: `${ testsRootPath }/test-results/results-data`,
globalSetup: require.resolve( './global-setup' ),
globalTeardown: require.resolve( './global-teardown' ),
testDir: 'tests',
testDir: `${ testsRootPath }/tests`,
retries: CI ? 2 : 0,
repeatEach: REPEAT_EACH ? Number( REPEAT_EACH ) : 1,
workers: 1,
@ -61,7 +64,7 @@ const config = {
use: {
baseURL: BASE_URL ?? 'http://localhost:8086',
screenshot: { mode: 'only-on-failure', fullPage: true },
stateDir: 'tests/e2e-pw/.state/',
stateDir: `${ testsRootPath }/tests/e2e-pw/.state/`,
trace: 'retain-on-failure',
video: 'retain-on-failure',
viewport: { width: 1280, height: 720 },
@ -69,18 +72,7 @@ const config = {
navigationTimeout: 20 * 1000,
},
snapshotPathTemplate: '{testDir}/{testFilePath}-snapshots/{arg}',
projects: [
{
name: 'default',
use: { ...devices[ 'Desktop Chrome' ] },
},
{
name: 'Gutenberg',
use: { ...devices[ 'Desktop Chrome' ] },
testIgnore:
/.*smoke-tests\/*|.*js-file-monitor\/*|.*admin-tasks\/*|.*activate-and-setup\/*|.*admin-analytics\/*|.*admin-marketing\/*|.*basic\/*|.*account-\/*|.*settings-\/*|.*users-\/*|.*order\/*|.*page-loads\/*/,
},
],
projects: [],
};
module.exports = config;

18
plugins/woocommerce/tests/e2e-pw/run-tests-with-env.sh Normal file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -eo pipefail
envName=$1
shift
SCRIPT_PATH=$(
cd "$(dirname "${BASH_SOURCE[0]}")" || return
pwd -P
)
echo "Setting up environment: $envName"
"$SCRIPT_PATH/envs/$envName/env-setup.sh"
echo "Running tests with environment: '$envName'"
echo "Arguments: $*"
pnpm playwright test --config="$SCRIPT_PATH"/envs/"$envName"/playwright.config.js "$@"