diff --git a/.github/workflows/cot-build-and-e2e-tests-daily.yml b/.github/workflows/cot-build-and-e2e-tests-daily.yml index ccea0f0dd25..b4beda9e3fc 100644 --- a/.github/workflows/cot-build-and-e2e-tests-daily.yml +++ b/.github/workflows/cot-build-and-e2e-tests-daily.yml @@ -1,20 +1,23 @@ name: Run daily tests in an environment with COT enabled on: - schedule: - - cron: "30 2 * * *" - workflow_dispatch: + schedule: + - cron: '30 2 * * *' + workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: cot-e2e-tests-run: name: Runs E2E tests with COT enabled. runs-on: ubuntu-20.04 + env: + ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results + ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report steps: - uses: actions/checkout@v3 - + - name: Setup WooCommerce Monorepo uses: ./.github/actions/setup-woocommerce-monorepo @@ -30,7 +33,7 @@ jobs: timeout-minutes: 60 id: run_playwright_e2e_tests env: - USE_WP_ENV: 1 + USE_WP_ENV: 1 working-directory: plugins/woocommerce run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js @@ -43,7 +46,7 @@ jobs: steps.run_playwright_e2e_tests.conclusion != 'skipped' ) working-directory: plugins/woocommerce - run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report + run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} - name: Archive Playwright E2E test report if: | @@ -53,8 +56,8 @@ jobs: with: name: e2e-test-report---pr-${{ github.event.number }} path: | - plugins/woocommerce/e2e/allure-results - plugins/woocommerce/e2e/allure-report + ${{ env.ALLURE_RESULTS_DIR }} + ${{ env.ALLURE_REPORT_DIR }} if-no-files-found: ignore retention-days: 5 @@ -62,10 +65,11 @@ jobs: name: Runs API tests with COT enabled. runs-on: ubuntu-20.04 env: - API_TEST_REPORT_DIR: ${{ github.workspace }}/api-test-report + ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results + ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report steps: - uses: actions/checkout@v3 - + - name: Setup WooCommerce Monorepo uses: ./.github/actions/setup-woocommerce-monorepo @@ -81,6 +85,7 @@ jobs: USER_KEY: admin USER_SECRET: password run: pnpm exec playwright test --config=tests/api-core-tests/playwright.config.js + - name: Generate Playwright API Test report. id: generate_api_report if: | @@ -90,7 +95,8 @@ jobs: steps.run_playwright_api_tests.conclusion != 'skipped' ) working-directory: plugins/woocommerce - run: pnpm exec allure generate --clean api-test-report/allure-results --output api-test-report/allure-report + run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} + - name: Archive Playwright API test report if: | always() && @@ -99,8 +105,8 @@ jobs: with: name: api-test-report---pr-${{ github.event.number }} path: | - plugins/woocommerce/api-test-report/allure-results - plugins/woocommerce/api-test-report/allure-report + ${{ env.ALLURE_RESULTS_DIR }} + ${{ env.ALLURE_REPORT_DIR }} if-no-files-found: ignore retention-days: 5 diff --git a/.github/workflows/cot-pr-build-and-e2e-tests.yml b/.github/workflows/cot-pr-build-and-e2e-tests.yml index 13ff7080081..1a1531deae8 100644 --- a/.github/workflows/cot-pr-build-and-e2e-tests.yml +++ b/.github/workflows/cot-pr-build-and-e2e-tests.yml @@ -13,9 +13,12 @@ jobs: name: Runs E2E tests with COT enabled. if: "${{ github.event_name == 'workflow_dispatch' || github.event.label.name == 'focus: custom order tables' }}" runs-on: ubuntu-20.04 + env: + ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results + ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report steps: - uses: actions/checkout@v3 - + - name: Setup WooCommerce Monorepo uses: ./.github/actions/setup-woocommerce-monorepo @@ -31,7 +34,7 @@ jobs: timeout-minutes: 60 id: run_playwright_e2e_tests env: - USE_WP_ENV: 1 + USE_WP_ENV: 1 working-directory: plugins/woocommerce run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js @@ -44,7 +47,7 @@ jobs: steps.run_playwright_e2e_tests.conclusion != 'skipped' ) working-directory: plugins/woocommerce - run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report + run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} - name: Archive Playwright E2E test report if: | @@ -54,8 +57,8 @@ jobs: with: name: e2e-test-report---pr-${{ github.event.number }} path: | - plugins/woocommerce/e2e/allure-results - plugins/woocommerce/e2e/allure-report + ${{ env.ALLURE_RESULTS_DIR }} + ${{ env.ALLURE_REPORT_DIR }} if-no-files-found: ignore retention-days: 5 @@ -64,10 +67,11 @@ jobs: if: "${{ github.event_name == 'workflow_dispatch' || github.event.label.name == 'focus: custom order tables' }}" runs-on: ubuntu-20.04 env: - API_TEST_REPORT_DIR: ${{ github.workspace }}/api-test-report + ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results + ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report steps: - uses: actions/checkout@v3 - + - name: Setup WooCommerce Monorepo uses: ./.github/actions/setup-woocommerce-monorepo @@ -93,8 +97,8 @@ jobs: steps.run_playwright_api_tests.conclusion != 'skipped' ) working-directory: plugins/woocommerce - run: pnpm exec allure generate --clean api-test-report/allure-results --output api-test-report/allure-report - + run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} + - name: Archive Playwright API test report if: | always() && @@ -103,8 +107,8 @@ jobs: with: name: api-test-report---pr-${{ github.event.number }} path: | - plugins/woocommerce/api-test-report/allure-results - plugins/woocommerce/api-test-report/allure-report + ${{ env.ALLURE_RESULTS_DIR }} + ${{ env.ALLURE_REPORT_DIR }} if-no-files-found: ignore retention-days: 5 diff --git a/.github/workflows/pr-build-and-e2e-tests.yml b/.github/workflows/pr-build-and-e2e-tests.yml index 00757a05801..f6a4c8e3f47 100644 --- a/.github/workflows/pr-build-and-e2e-tests.yml +++ b/.github/workflows/pr-build-and-e2e-tests.yml @@ -11,6 +11,9 @@ jobs: e2e-tests-run: name: Runs E2E tests. runs-on: ubuntu-20.04 + env: + ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results + ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report outputs: E2E_GRAND_TOTAL: ${{ steps.count_e2e_total.outputs.E2E_GRAND_TOTAL }} steps: @@ -40,9 +43,9 @@ jobs: timeout-minutes: 60 id: run_playwright_e2e_tests env: - USE_WP_ENV: 1 - E2E_MAX_FAILURES: 15 - FORCE_COLOR: 1 + USE_WP_ENV: 1 + E2E_MAX_FAILURES: 15 + FORCE_COLOR: 1 working-directory: plugins/woocommerce run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js @@ -55,7 +58,7 @@ jobs: steps.run_playwright_e2e_tests.conclusion != 'skipped' ) working-directory: plugins/woocommerce - run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report + run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} - name: Archive Playwright E2E test report if: | @@ -65,8 +68,8 @@ jobs: with: name: e2e-test-report---pr-${{ github.event.number }} path: | - plugins/woocommerce/e2e/allure-results - plugins/woocommerce/e2e/allure-report + ${{ env.ALLURE_RESULTS_DIR }} + ${{ env.ALLURE_REPORT_DIR }} if-no-files-found: ignore retention-days: 5 @@ -74,7 +77,8 @@ jobs: name: Runs API tests. runs-on: ubuntu-20.04 env: - API_TEST_REPORT_DIR: ${{ github.workspace }}/api-test-report + ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results + ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report steps: - uses: actions/checkout@v3 @@ -103,7 +107,7 @@ jobs: steps.run_playwright_api_tests.conclusion != 'skipped' ) working-directory: plugins/woocommerce - run: pnpm exec allure generate --clean api-test-report/allure-results --output api-test-report/allure-report + run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} - name: Archive Playwright API test report if: | always() && @@ -112,8 +116,8 @@ jobs: with: name: api-test-report---pr-${{ github.event.number }} path: | - plugins/woocommerce/api-test-report/allure-results - plugins/woocommerce/api-test-report/allure-report + ${{ env.ALLURE_RESULTS_DIR }} + ${{ env.ALLURE_REPORT_DIR }} if-no-files-found: ignore retention-days: 5 diff --git a/.github/workflows/smoke-test-daily.yml b/.github/workflows/smoke-test-daily.yml index 7774735db14..640cd61cd27 100644 --- a/.github/workflows/smoke-test-daily.yml +++ b/.github/workflows/smoke-test-daily.yml @@ -68,7 +68,7 @@ jobs: steps.e2e.conclusion != 'skipped' ) working-directory: plugins/woocommerce - run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report + run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} - name: Archive E2E test report if: | @@ -78,8 +78,8 @@ jobs: with: name: ${{ env.E2E_ARTIFACT }} path: | - plugins/woocommerce/e2e/allure-results - plugins/woocommerce/e2e/allure-report + ${{ env.ALLURE_RESULTS_DIR }} + ${{ env.ALLURE_REPORT_DIR }} if-no-files-found: ignore retention-days: 5 @@ -88,6 +88,9 @@ jobs: runs-on: ubuntu-20.04 needs: [e2e-tests] if: always() + env: + ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results + ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report steps: - uses: actions/checkout@v3 with: @@ -119,7 +122,7 @@ jobs: steps.run_playwright_api_tests.conclusion != 'skipped' ) working-directory: plugins/woocommerce - run: pnpm exec allure generate --clean api-test-report/allure-results --output api-test-report/allure-report + run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} - name: Archive API test report if: | @@ -129,8 +132,8 @@ jobs: with: name: ${{ env.API_ARTIFACT }} path: | - plugins/woocommerce/api-test-report/allure-results - plugins/woocommerce/api-test-report/allure-report + ${{ env.ALLURE_RESULTS_DIR }} + ${{ env.ALLURE_REPORT_DIR }} if-no-files-found: ignore retention-days: 5 @@ -194,6 +197,8 @@ jobs: if: always() env: USE_WP_ENV: 1 + ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results + ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report strategy: fail-fast: false matrix: @@ -254,7 +259,7 @@ jobs: steps.e2e.conclusion != 'skipped' ) working-directory: plugins/woocommerce - run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report + run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} - name: Archive E2E test report if: | @@ -264,8 +269,8 @@ jobs: with: name: Smoke tests with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) path: | - plugins/woocommerce/e2e/allure-results - plugins/woocommerce/e2e/allure-report + ${{ env.ALLURE_RESULTS_DIR }} + ${{ env.ALLURE_REPORT_DIR }} if-no-files-found: ignore retention-days: 5 diff --git a/plugins/woocommerce/changelog/e2e-use-e2e-pw-folder-for-outputs b/plugins/woocommerce/changelog/e2e-use-e2e-pw-folder-for-outputs new file mode 100644 index 00000000000..6ef49004835 --- /dev/null +++ b/plugins/woocommerce/changelog/e2e-use-e2e-pw-folder-for-outputs @@ -0,0 +1,4 @@ +Significance: patch +Type: dev + +Use plugins/woocommerce/tests/e2e-pw folder for saving test outputs diff --git a/plugins/woocommerce/tests/api-core-tests/playwright.config.js b/plugins/woocommerce/tests/api-core-tests/playwright.config.js index caf5638cbb2..ed1ecd645b6 100644 --- a/plugins/woocommerce/tests/api-core-tests/playwright.config.js +++ b/plugins/woocommerce/tests/api-core-tests/playwright.config.js @@ -34,9 +34,19 @@ const config = { ], [ 'allure-playwright', - { outputFolder: 'api-test-report/allure-results' }, + { + outputFolder: + process.env.ALLURE_RESULTS_DIR ?? + 'tests/api-core-tests/api-test-report/allure-results', + }, + ], + [ + 'json', + { + outputFile: + 'tests/api-core-tests/api-test-report/test-results.json', + }, ], - [ 'json', { outputFile: 'api-test-report/test-results.json' } ], ], use: { screenshot: 'only-on-failure', diff --git a/plugins/woocommerce/tests/e2e-pw/playwright.config.js b/plugins/woocommerce/tests/e2e-pw/playwright.config.js index 07db8f7bda0..f218c67baf8 100644 --- a/plugins/woocommerce/tests/e2e-pw/playwright.config.js +++ b/plugins/woocommerce/tests/e2e-pw/playwright.config.js @@ -1,9 +1,10 @@ const { devices } = require( '@playwright/test' ); const { - CI, - E2E_MAX_FAILURES, + ALLURE_RESULTS_DIR, BASE_URL, + CI, DEFAULT_TIMEOUT_OVERRIDE, + E2E_MAX_FAILURES, } = process.env; const config = { @@ -26,17 +27,23 @@ const config = { open: CI ? 'never' : 'always', }, ], - [ 'allure-playwright', { outputFolder: 'e2e/allure-results' } ], - [ 'json', { outputFile: 'e2e/test-results.json' } ], + [ + 'allure-playwright', + { + outputFolder: + ALLURE_RESULTS_DIR ?? 'tests/e2e-pw/allure-results', + }, + ], + [ 'json', { outputFile: 'tests/e2e-pw/test-results.json' } ], ], maxFailures: E2E_MAX_FAILURES ? Number( E2E_MAX_FAILURES ) : 0, use: { - screenshot: 'only-on-failure', - video: 'on-first-retry', - trace: 'retain-on-failure', - viewport: { width: 1280, height: 720 }, baseURL: BASE_URL ?? 'http://localhost:8086', - stateDir: 'e2e/storage/', + screenshot: 'only-on-failure', + stateDir: 'tests/e2e-pw/storage/', + trace: 'retain-on-failure', + video: 'on-first-retry', + viewport: { width: 1280, height: 720 }, }, projects: [ { diff --git a/plugins/woocommerce/tests/e2e-pw/storage/adminState.json b/plugins/woocommerce/tests/e2e-pw/storage/adminState.json deleted file mode 100644 index 8098ee78c77..00000000000 --- a/plugins/woocommerce/tests/e2e-pw/storage/adminState.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "cookies": [ - { - "sameSite": "Lax", - "name": "wordpress_dc5025de8b60c0a511df7c07d81ead97", - "value": "admin%7C1660405922%7CjsNIRwmbjFh7KIGlSHy50rSs6X0L0zISppmBqgh0u0u%7C8ac4ff905331544a3cbcf0c58840ebfc3f492d4511dd17f48dbed18f5662a2c6", - "domain": "localhost", - "path": "/wp-content/plugins", - "expires": -1, - "httpOnly": true, - "secure": false - }, - { - "sameSite": "Lax", - "name": "wordpress_dc5025de8b60c0a511df7c07d81ead97", - "value": "admin%7C1660405922%7CjsNIRwmbjFh7KIGlSHy50rSs6X0L0zISppmBqgh0u0u%7C8ac4ff905331544a3cbcf0c58840ebfc3f492d4511dd17f48dbed18f5662a2c6", - "domain": "localhost", - "path": "/wp-admin", - "expires": -1, - "httpOnly": true, - "secure": false - }, - { - "sameSite": "Lax", - "name": "wordpress_test_cookie", - "value": "WP%20Cookie%20check", - "domain": "localhost", - "path": "/", - "expires": -1, - "httpOnly": false, - "secure": false - }, - { - "sameSite": "Lax", - "name": "wordpress_logged_in_dc5025de8b60c0a511df7c07d81ead97", - "value": "admin%7C1660405922%7CjsNIRwmbjFh7KIGlSHy50rSs6X0L0zISppmBqgh0u0u%7C6de27e0d8393de7715c07e144424d90733d53e65b8665c8a7db1ad94798c369c", - "domain": "localhost", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": false - }, - { - "sameSite": "Lax", - "name": "tk_ai", - "value": "woo%3AT9QFbJmpuSGWcjApaONaBtSB", - "domain": "localhost", - "path": "/", - "expires": -1, - "httpOnly": false, - "secure": false - }, - { - "sameSite": "Lax", - "name": "wp-settings-time-1", - "value": "1660233126", - "domain": "localhost", - "path": "/", - "expires": 1691769126.937059, - "httpOnly": false, - "secure": false - } - ], - "origins": [] -} \ No newline at end of file diff --git a/plugins/woocommerce/tests/e2e-pw/storage/customerState.json b/plugins/woocommerce/tests/e2e-pw/storage/customerState.json deleted file mode 100644 index 6fa41f487e0..00000000000 --- a/plugins/woocommerce/tests/e2e-pw/storage/customerState.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "cookies": [ - { - "sameSite": "Lax", - "name": "wordpress_dc5025de8b60c0a511df7c07d81ead97", - "value": "customer%7C1660405929%7CL0OCLRBBMubq8iKqOeQU1NqOzTWFd7ppECd1n100GKG%7Cccb6b6b9e1190e94ef751360fded6ec026c89350f9f4695e9bf900ca09d72a84", - "domain": "localhost", - "path": "/wp-content/plugins", - "expires": -1, - "httpOnly": true, - "secure": false - }, - { - "sameSite": "Lax", - "name": "wordpress_dc5025de8b60c0a511df7c07d81ead97", - "value": "customer%7C1660405929%7CL0OCLRBBMubq8iKqOeQU1NqOzTWFd7ppECd1n100GKG%7Cccb6b6b9e1190e94ef751360fded6ec026c89350f9f4695e9bf900ca09d72a84", - "domain": "localhost", - "path": "/wp-admin", - "expires": -1, - "httpOnly": true, - "secure": false - }, - { - "sameSite": "Lax", - "name": "wordpress_test_cookie", - "value": "WP%20Cookie%20check", - "domain": "localhost", - "path": "/", - "expires": -1, - "httpOnly": false, - "secure": false - }, - { - "sameSite": "Lax", - "name": "wordpress_logged_in_dc5025de8b60c0a511df7c07d81ead97", - "value": "customer%7C1660405929%7CL0OCLRBBMubq8iKqOeQU1NqOzTWFd7ppECd1n100GKG%7C869a28fa8ef17799b6d097d84c8155c7d0dd18cd9967f7a18e3f49f41dd71226", - "domain": "localhost", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": false - } - ], - "origins": [] -} \ No newline at end of file