[blocks e2e] Move Blocks e2e into ci.yml (#48224)
This commit is contained in:
Adrian Moldovan
GitHub
parent
5650f88514
commit
a7f634d5ef
|
|
@ -163,14 +163,26 @@ jobs:
|
|||
CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} # required by Metrics tests
|
||||
run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
|
||||
|
||||
- name: 'Resolve artifacts path'
|
||||
# Blocks e2e use a relative path which is not supported by actions/upload-artifact@v4
|
||||
# https://github.com/actions/upload-artifact/issues/176
|
||||
env:
|
||||
ARTIFACTS_PATH: '${{ matrix.projectPath }}/${{ matrix.report.resultsPath }}'
|
||||
run: echo "ARTIFACTS_PATH=$(realpath $ARTIFACTS_PATH)" >> $GITHUB_ENV
|
||||
|
||||
- name: 'Upload artifacts'
|
||||
if: ${{ always() && matrix.report.resultsPath != '' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: '${{ matrix.report.resultsBlobName }}-${{ strategy.job-index }}'
|
||||
path: '${{ matrix.projectPath }}/${{ matrix.report.resultsPath }}'
|
||||
retention-days: 1
|
||||
compression-level: 9
|
||||
path: ${{ env.ARTIFACTS_PATH }}
|
||||
|
||||
- name: 'Upload flaky test reports'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: flaky-tests-${{ strategy.job-index }}
|
||||
path: ${{ env.ARTIFACTS_PATH }}/flaky-tests
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: 'Archive metrics results'
|
||||
if: ${{ success() && startsWith(matrix.name, 'Metrics') }} # this seems too fragile, we should update the reporting path and use the generic upload step above
|
||||
|
|
@ -338,3 +350,49 @@ jobs:
|
|||
if: ${{ github.event_name == 'pull_request' && contains(matrix.report, 'e2e') }}
|
||||
run: |
|
||||
echo "::notice::🔗🔗 The e2e report for this run is available at https://woocommerce.github.io/woocommerce-test-reports/pr/${{ github.event.pull_request.number }}/e2e"
|
||||
|
||||
|
||||
report-flaky-tests:
|
||||
name: 'Create issues for flaky tests'
|
||||
if: ${{ !cancelled() && ! github.event.pull_request.head.repo.fork }}
|
||||
needs: ['project-test-jobs']
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
repository: WordPress/gutenberg
|
||||
ref: dbf201449e9736f672b61e422787d47659db327a
|
||||
|
||||
- uses: actions/download-artifact@v4
|
||||
id: download-artifact
|
||||
with:
|
||||
pattern: flaky-tests*
|
||||
path: flaky-tests
|
||||
merge-multiple: true
|
||||
|
||||
- name: 'Check if there are flaky tests reports'
|
||||
run: |
|
||||
downloadPath='${{ steps.download-artifact.outputs.download-path }}'
|
||||
# make dir so that next step doesn't fail if it doesn't exist
|
||||
mkdir -p $downloadPath
|
||||
# any output means there are reports
|
||||
echo "FLAKY_REPORTS=$(ls -A $downloadPath | head -1)" >> $GITHUB_ENV
|
||||
|
||||
- name: 'Setup'
|
||||
if: ${{ !!env.FLAKY_REPORTS }}
|
||||
uses: ./.github/setup-node
|
||||
|
||||
- name: 'Build packages'
|
||||
if: ${{ !!env.FLAKY_REPORTS }}
|
||||
run: npm run build:packages
|
||||
|
||||
- name: 'Report flaky tests'
|
||||
if: ${{ !!env.FLAKY_REPORTS }}
|
||||
uses: ./packages/report-flaky-tests
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
label: 'metric: flaky e2e test'
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ const IGNORED_JOBS = [
|
|||
/Evaluate Project Job Statuses/,
|
||||
/Report results on Slack/,
|
||||
/Publish reports/,
|
||||
/Create issues for flaky tests/,
|
||||
];
|
||||
|
||||
const isJobRequired = ( job ) => {
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@
|
|||
"test:e2e:block-theme": "pnpm run test:e2e block_theme",
|
||||
"test:e2e:classic-theme": "pnpm run test:e2e classic_theme",
|
||||
"test:e2e:block-theme-with-templates": "pnpm run test:e2e block_theme_with_templates",
|
||||
"test:e2e:fake": "echo 'It should trigger Blocks e2e tests in @woocommerce/plugin-woocommerce'",
|
||||
"test:e2e:jest": "pnpm run wp-env:config && cross-env JEST_PUPPETEER_CONFIG=tests/e2e-jest/config/jest-puppeteer.config.js NODE_CONFIG_DIR=tests/e2e-jest/config wp-scripts test-e2e --config tests/e2e-jest/config/jest.config.js",
|
||||
"test:e2e:jest:dev": "pnpm run wp-env:config && cross-env JEST_PUPPETEER_CONFIG=tests/e2e-jest/config/jest-puppeteer.config-dev.js NODE_CONFIG_DIR=tests/e2e-jest/config wp-scripts test-e2e --config tests/e2e-jest/config/jest.config.js",
|
||||
"test:e2e:jest:dev-watch": "pnpm run wp-env:config && cross-env JEST_PUPPETEER_CONFIG=tests/e2e-jest/config/jest-puppeteer.config-dev.js NODE_CONFIG_DIR=tests/e2e-jest/config wp-scripts test-e2e --config tests/e2e-jest/config/jest.config.js --watch",
|
||||
|
|
@ -370,6 +371,17 @@
|
|||
"pull_request",
|
||||
"push"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Blocks e2e - trigger for Blocks e2e tests in @woocommerce/plugin-woocommerce",
|
||||
"testType": "e2e",
|
||||
"command": "test:e2e:fake",
|
||||
"changes": [
|
||||
"tests/e2e/**"
|
||||
],
|
||||
"events": [
|
||||
"pull_request"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,13 @@
|
|||
import fs from 'fs';
|
||||
import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter';
|
||||
|
||||
/**
|
||||
* Internal dependencies
|
||||
*/
|
||||
import pwConfig from './playwright.config';
|
||||
|
||||
type FormattedTestResult = Omit< TestResult, 'steps' >;
|
||||
const flakyTestsDir = `${ pwConfig.outputDir }/flaky-tests`;
|
||||
|
||||
// Remove "steps" to prevent stringify circular structure.
|
||||
function formatTestResult( testResult: TestResult ): FormattedTestResult {
|
||||
|
|
@ -26,7 +32,9 @@ class FlakyTestsReporter implements Reporter {
|
|||
|
||||
onBegin() {
|
||||
try {
|
||||
fs.mkdirSync( 'flaky-tests' );
|
||||
fs.mkdirSync( flakyTestsDir, {
|
||||
recursive: true,
|
||||
} );
|
||||
} catch ( err ) {
|
||||
if (
|
||||
err instanceof Error &&
|
||||
|
|
@ -56,7 +64,7 @@ class FlakyTestsReporter implements Reporter {
|
|||
case 'flaky': {
|
||||
const safeFileName = testTitle.replace( /[^a-z0-9]/gi, '_' );
|
||||
fs.writeFileSync(
|
||||
`flaky-tests/${ safeFileName }.json`,
|
||||
`${ flakyTestsDir }/${ safeFileName }.json`,
|
||||
JSON.stringify( {
|
||||
version: 1,
|
||||
runner: '@playwright/test',
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ const { CI, DEFAULT_TIMEOUT_OVERRIDE } = process.env;
|
|||
const config: PlaywrightTestConfig = {
|
||||
maxFailures: 0,
|
||||
timeout: parseInt( DEFAULT_TIMEOUT_OVERRIDE || '', 10 ) || 100_000, // Defaults to 100s.
|
||||
outputDir: './artifacts/test-results',
|
||||
outputDir: `${ __dirname }/artifacts/test-results`,
|
||||
globalSetup: fileURLToPath(
|
||||
new URL( 'global-setup.ts', 'file:' + __filename ).href
|
||||
),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
Significance: patch
|
||||
Type: dev
|
||||
|
||||
Include blocks e2e in ci.yml
|
||||
|
|
@ -27,6 +27,7 @@
|
|||
"env:performance-init": "./tests/performance/bin/init-sample-products.sh",
|
||||
"env:restart": "pnpm wp-env destroy && pnpm wp-env start --update",
|
||||
"env:start": "pnpm wp-env start",
|
||||
"env:start:blocks": "pnpm --filter='@woocommerce/block-library' env:start && pnpm playwright install chromium --with-deps",
|
||||
"env:stop": "pnpm wp-env stop",
|
||||
"env:test": "pnpm env:dev && pnpm playwright install chromium",
|
||||
"env:perf:install-k6": "curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1",
|
||||
|
|
@ -54,6 +55,7 @@
|
|||
"test:api-pw": "USE_WP_ENV=1 pnpm playwright test --config=tests/api-core-tests/playwright.config.js",
|
||||
"test:e2e-pw": "pnpm playwright test --config=tests/e2e-pw/envs/default/playwright.config.js",
|
||||
"test:e2e": "pnpm test:e2e:with-env default",
|
||||
"test:e2e:blocks": "pnpm --filter='@woocommerce/block-library' test:e2e",
|
||||
"test:e2e:with-env": "bash ./tests/e2e-pw/run-tests-with-env.sh",
|
||||
"test:perf": "./k6 run ./tests/performance/tests/gh-action-pr-requests.js",
|
||||
"test:env:start": "pnpm env:test",
|
||||
|
|
@ -513,6 +515,40 @@
|
|||
"pull_request",
|
||||
"push"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Blocks e2e tests",
|
||||
"testType": "e2e",
|
||||
"command": "test:e2e:blocks",
|
||||
"shardingArguments": [
|
||||
"--shard=1/10",
|
||||
"--shard=2/10",
|
||||
"--shard=3/10",
|
||||
"--shard=4/10",
|
||||
"--shard=5/10",
|
||||
"--shard=6/10",
|
||||
"--shard=7/10",
|
||||
"--shard=8/10",
|
||||
"--shard=9/10",
|
||||
"--shard=10/10"
|
||||
],
|
||||
"changes": [
|
||||
"src/Blocks/**",
|
||||
"templates/**",
|
||||
"patterns/**"
|
||||
],
|
||||
"testEnv": {
|
||||
"start": "env:start:blocks"
|
||||
},
|
||||
"events": [
|
||||
"pull_request",
|
||||
"push",
|
||||
"release-checks"
|
||||
],
|
||||
"report": {
|
||||
"resultsBlobName": "blocks-e2e-report",
|
||||
"resultsPath": "../woocommerce-blocks/tests/e2e/artifacts/test-results"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue