woocommerce/plugins/woocommerce-blocks/tests/e2e/playwright.config.ts

58 lines
1.5 KiB
TypeScript
Raw Normal View History

/**
* External dependencies
*/
import { fileURLToPath } from 'url';
import { BASE_URL, STORAGE_STATE_PATH } from '@woocommerce/e2e-utils';
import { PlaywrightTestConfig, defineConfig, devices } from '@playwright/test';
const { CI, DEFAULT_TIMEOUT_OVERRIDE } = process.env;
const config: PlaywrightTestConfig = {
maxFailures: CI ? 30 : 0,
timeout: parseInt( DEFAULT_TIMEOUT_OVERRIDE || '', 10 ) || 100_000, // Defaults to 100s.
outputDir: `${ __dirname }/artifacts/test-results`,
globalSetup: fileURLToPath(
new URL( 'global-setup.ts', 'file:' + __filename ).href
),
testDir: './tests',
retries: CI ? 1 : 0,
workers: 1,
reportSlowTests: { max: 5, threshold: 30 * 1000 }, // 30 seconds threshold
fullyParallel: false,
forbidOnly: !! CI,
WooCommerce Blocks: Added a GitHub Action to create issues for flaky E2E tests (#47758) * Add flaky test reporting and issue creation for Blocks This commit introduces the functionality for identifying and reporting flaky tests within our CI workflow. - The reporter captures flaky test results and saves them to `flaky-tests` directory. - If a test fails initially but passes upon retries, it is marked as flaky and a corresponding report is generated. - The workflow creates GitHub issues for flaky tests, aiding in better visibility and tracking of these intermittent issues. 1. **Workflow Changes**: - **Archive flaky test reports**: Adds a step to archive flaky test reports generated during the test runs. - **Merge flaky test reports**: Introduces a new step to merge all flaky test reports from different shards. - **Create GitHub issues for flaky tests**: Implements a job that uses the `@wordpress/report-flaky-tests` package to create GitHub issues for detected flaky tests when a PR is not in draft state and flaky test reports are present. 2. **New Flaky Test Reporter**: - Adds `flaky-tests-reporter.ts` that defines a custom Playwright reporter for identifying and recording flaky tests. 3. **Playwright Configuration**: - Updates `playwright.config.ts` to include the new flaky tests reporter in the list of reporters when running in CI. 4. **Demo Tests**: - Adds `demo.spec.ts` containing sample tests to validate the flaky test reporting functionality. Implementing this feature will help us track and address flaky tests more efficiently, ensuring more stable and reliable test suites. * Remove draft condition for testing * 'Merge failures artifacts' job shouldn't touch flaky test artifacts * Try fixing if condition * Add required permissions * Let's try providing all the permissions * Let's try providing all the permissions * Create flaky tests issues when PR is merged with trunk * Update if condition * Run action for specific paths only * Add changefile(s) from automation for the following project(s): woocommerce-blocks * Remove Demo file * Remove Demo file * Use commit hash to use exact version of the script * Replace label with team: Kirigami & Origami --------- Co-authored-by: github-actions <github-actions@github.com>
2024-06-05 14:19:28 +00:00
reporter: process.env.CI
? [
[ 'list' ],
[ './flaky-tests-reporter.ts' ],
[
'allure-playwright',
{
outputFolder: `${ __dirname }/artifacts/test-results/allure-results`,
},
],
[ 'buildkite-test-collector/playwright/reporter' ],
]
WooCommerce Blocks: Added a GitHub Action to create issues for flaky E2E tests (#47758) * Add flaky test reporting and issue creation for Blocks This commit introduces the functionality for identifying and reporting flaky tests within our CI workflow. - The reporter captures flaky test results and saves them to `flaky-tests` directory. - If a test fails initially but passes upon retries, it is marked as flaky and a corresponding report is generated. - The workflow creates GitHub issues for flaky tests, aiding in better visibility and tracking of these intermittent issues. 1. **Workflow Changes**: - **Archive flaky test reports**: Adds a step to archive flaky test reports generated during the test runs. - **Merge flaky test reports**: Introduces a new step to merge all flaky test reports from different shards. - **Create GitHub issues for flaky tests**: Implements a job that uses the `@wordpress/report-flaky-tests` package to create GitHub issues for detected flaky tests when a PR is not in draft state and flaky test reports are present. 2. **New Flaky Test Reporter**: - Adds `flaky-tests-reporter.ts` that defines a custom Playwright reporter for identifying and recording flaky tests. 3. **Playwright Configuration**: - Updates `playwright.config.ts` to include the new flaky tests reporter in the list of reporters when running in CI. 4. **Demo Tests**: - Adds `demo.spec.ts` containing sample tests to validate the flaky test reporting functionality. Implementing this feature will help us track and address flaky tests more efficiently, ensuring more stable and reliable test suites. * Remove draft condition for testing * 'Merge failures artifacts' job shouldn't touch flaky test artifacts * Try fixing if condition * Add required permissions * Let's try providing all the permissions * Let's try providing all the permissions * Create flaky tests issues when PR is merged with trunk * Update if condition * Run action for specific paths only * Add changefile(s) from automation for the following project(s): woocommerce-blocks * Remove Demo file * Remove Demo file * Use commit hash to use exact version of the script * Replace label with team: Kirigami & Origami --------- Co-authored-by: github-actions <github-actions@github.com>
2024-06-05 14:19:28 +00:00
: 'list',
use: {
baseURL: BASE_URL,
screenshot: 'only-on-failure',
trace:
/^https?:\/\/localhost/.test( BASE_URL ) || ! CI
? 'retain-on-first-failure'
: 'off',
video: 'on-first-retry',
viewport: { width: 1280, height: 720 },
storageState: STORAGE_STATE_PATH,
actionTimeout: 10_000,
Fix E2E tests for WordPress 6.5 (#44593) * Add sharding to Blocks e2e tests * Changelog * Push a change to Blocks README to trigger tests * Fix syntax * Fixed again * Try again * Trying to get the shard number * Shard names * fix E2E test * use wordpress 6.5 as test env * update util * fix Mini Cart Block E2E test * add editor dependency * fix E2E tests * fix E2E tests * fix E2E tests * restore global setup file * fix E2E tests * fix global setup * try now * try now * fix E2E test * fix E2E test * fix mini cart E2E test * add changelog * complete all the jobs * update playwright * fix E2E tests * fix E2E test * try now * fix E2E test * not show welcome guide for the post editor * update configuration * fix guest storage state * fix E2E tests * check now * fix E2E tests * fix command * improve selector * use WordPress 6.5 beta 2 * use WordPress 6.5 beta 3 * fix canvas use * wait for button * fix E2E test * fix E2E test * fix product button * fix product collection tests * fix wp build link * use beforeEach * fix E2E tests * use WordPress RC2 * Add changefile(s) from automation for the following project(s): woocommerce-blocks, woocommerce * trigger CI * fix E2E tests * fix E2E tests * fix E2E test * fix E2E tests * add timeout * Fix flaky locator assertions ...by replacing them with dedicaed auto-waiting ones. * Apply correct use of waitForRequest/Response and remove where unnecessary * Remove obsolete waitUntils * Fix accidentally isolated test * Replace saveTemplate with core's saveSiteEditorEntities * Fix findProductRow util * Restore necessary request waiters * Use activateTheme util instead of cli call * Remove unused import * Fix request destructured call Request properties cannot be destructured (todo: create upstream issue) * Revert some waitForRequest changes These need to be investigated for heavy bleeding from side-effects * Fix failing cart checkout test * Set navigation timeout to 10s We need this so that API like page.goto or page.waitForURL doesn't take 100 seconds (global timeout) to throw. * Do not close the page in the page fixture override The page will be closed anyway, and by closing it explicitly in the teardown phase of the fixture we create a flaky race condition where the page might be closed before the action/navigation timeout is reached, resulting in "page closed!" error instead of, e.g. "locator.fill timed out". * Revert product collection tests due to conflicts * Try fixing E2E test * Remove some random change * Revert to programmatic blur :shrug: * Fix additional checkout fields test * Add a waitForRequest safety check in case it's debounced * Fix price filter test * Fix flaky visitTemplateEditor util * Remove obsolete code from global setup * Revert "Fix flaky visitTemplateEditor util" This reverts commit 227336f2653eb17b37ade45634374d69ef221c50. * update to WordPress RC3 * Add waitForCustomerDataUpdate function * Don't wait for requests and use data store instead * Try with new wait condition * Re-add wait condition * Wait for customer update in guest checkout test * Add changelog * fix lint error * fix some E2E tests * disable welcome guide * remove beforeAll * add timeout * add another timeout * increase timeout * Add call to waitForSiteEditorFinishLoading when editing the theme template * remove timeout * remove not necessary comment * clean changelog * Add changefile(s) from automation for the following project(s): woocommerce-blocks, woocommerce * remove changelog --------- Co-authored-by: Jon Lane <jon.lane@automattic.com> Co-authored-by: github-actions <github-actions@github.com> Co-authored-by: Bart Kalisz <bartlomiej.kalisz@gmail.com> Co-authored-by: Karol Manijak <20098064+kmanijak@users.noreply.github.com> Co-authored-by: Thomas Roberts <thomas.roberts@automattic.com> Co-authored-by: Albert Juhé Lluveras <contact@albertjuhe.com>
2024-03-27 09:54:44 +00:00
navigationTimeout: 10_000,
},
projects: [
{
name: 'chromium',
use: { ...devices[ 'Desktop Chrome' ] },
},
],
};
export default defineConfig( config );