Set paths for `allure-results`, `test-results.json`, and save state files to be inside their respective E2E or API folders (#35206)

* Set paths to e2e-pw

* Delete and untrack storage state files

* Add changelog

* Checkout updated version of smoke test daily workflow

* Allow setting allure output paths to e2e-pw folder using environment variables

* Set allure output paths to be inside api-core-tests/api-test-report folder

* Remove unnecessary TODO comment

.github/workflows/cot-build-and-e2e-tests-daily.yml

@@ -1,7 +1,7 @@
 name: Run daily tests in an environment with COT enabled
 on:
     schedule:
-    - cron: "30 2 * * *"
+        - cron: '30 2 * * *'
     workflow_dispatch:
 
 concurrency:
@@ -12,6 +12,9 @@ jobs:
     cot-e2e-tests-run:
         name: Runs E2E tests with COT enabled.
         runs-on: ubuntu-20.04
+        env:
+            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results
+            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report
         steps:
             - uses: actions/checkout@v3
 
@@ -43,7 +46,7 @@ jobs:
                     steps.run_playwright_e2e_tests.conclusion != 'skipped' 
                   )
               working-directory: plugins/woocommerce
-              run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report
+              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
 
             - name: Archive Playwright E2E test report
               if: |
@@ -53,8 +56,8 @@ jobs:
               with:
                   name: e2e-test-report---pr-${{ github.event.number }}
                   path: |
-                      plugins/woocommerce/e2e/allure-results
+                      ${{ env.ALLURE_RESULTS_DIR }}
-                      plugins/woocommerce/e2e/allure-report
+                      ${{ env.ALLURE_REPORT_DIR }}
                   if-no-files-found: ignore
                   retention-days: 5
 
@@ -62,7 +65,8 @@ jobs:
         name: Runs API tests with COT enabled.
         runs-on: ubuntu-20.04
         env:
-            API_TEST_REPORT_DIR: ${{ github.workspace }}/api-test-report
+            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results
+            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report
         steps:
             - uses: actions/checkout@v3
 
@@ -81,6 +85,7 @@ jobs:
                   USER_KEY: admin
                   USER_SECRET: password
               run: pnpm exec playwright test --config=tests/api-core-tests/playwright.config.js
+
             - name: Generate Playwright API Test report.
               id: generate_api_report
               if: |
@@ -90,7 +95,8 @@ jobs:
                     steps.run_playwright_api_tests.conclusion != 'skipped' 
                   )
               working-directory: plugins/woocommerce
-              run: pnpm exec allure generate --clean api-test-report/allure-results --output api-test-report/allure-report
+              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
+
             - name: Archive Playwright API test report
               if: |
                   always() &&
@@ -99,8 +105,8 @@ jobs:
               with:
                   name: api-test-report---pr-${{ github.event.number }}
                   path: |
-                      plugins/woocommerce/api-test-report/allure-results
+                      ${{ env.ALLURE_RESULTS_DIR }}
-                      plugins/woocommerce/api-test-report/allure-report
+                      ${{ env.ALLURE_REPORT_DIR }}
                   if-no-files-found: ignore
                   retention-days: 5
 

.github/workflows/cot-pr-build-and-e2e-tests.yml

@@ -13,6 +13,9 @@ jobs:
         name: Runs E2E tests with COT enabled.
         if: "${{ github.event_name == 'workflow_dispatch' || github.event.label.name == 'focus: custom order tables' }}"
         runs-on: ubuntu-20.04
+        env:
+            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results
+            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report
         steps:
             - uses: actions/checkout@v3
 
@@ -44,7 +47,7 @@ jobs:
                     steps.run_playwright_e2e_tests.conclusion != 'skipped' 
                   )
               working-directory: plugins/woocommerce
-              run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report
+              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
 
             - name: Archive Playwright E2E test report
               if: |
@@ -54,8 +57,8 @@ jobs:
               with:
                   name: e2e-test-report---pr-${{ github.event.number }}
                   path: |
-                      plugins/woocommerce/e2e/allure-results
+                      ${{ env.ALLURE_RESULTS_DIR }}
-                      plugins/woocommerce/e2e/allure-report
+                      ${{ env.ALLURE_REPORT_DIR }}
                   if-no-files-found: ignore
                   retention-days: 5
 
@@ -64,7 +67,8 @@ jobs:
         if: "${{ github.event_name == 'workflow_dispatch' || github.event.label.name == 'focus: custom order tables' }}"
         runs-on: ubuntu-20.04
         env:
-            API_TEST_REPORT_DIR: ${{ github.workspace }}/api-test-report
+            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results
+            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report
         steps:
             - uses: actions/checkout@v3
 
@@ -93,7 +97,7 @@ jobs:
                     steps.run_playwright_api_tests.conclusion != 'skipped' 
                   )
               working-directory: plugins/woocommerce
-              run: pnpm exec allure generate --clean api-test-report/allure-results --output api-test-report/allure-report
+              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
 
             - name: Archive Playwright API test report
               if: |
@@ -103,8 +107,8 @@ jobs:
               with:
                   name: api-test-report---pr-${{ github.event.number }}
                   path: |
-                      plugins/woocommerce/api-test-report/allure-results
+                      ${{ env.ALLURE_RESULTS_DIR }}
-                      plugins/woocommerce/api-test-report/allure-report
+                      ${{ env.ALLURE_REPORT_DIR }}
                   if-no-files-found: ignore
                   retention-days: 5
 

.github/workflows/pr-build-and-e2e-tests.yml

@@ -11,6 +11,9 @@ jobs:
     e2e-tests-run:
         name: Runs E2E tests.
         runs-on: ubuntu-20.04
+        env:
+            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results
+            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report
         outputs:
             E2E_GRAND_TOTAL: ${{ steps.count_e2e_total.outputs.E2E_GRAND_TOTAL }}
         steps:
@@ -55,7 +58,7 @@ jobs:
                     steps.run_playwright_e2e_tests.conclusion != 'skipped' 
                   )
               working-directory: plugins/woocommerce
-              run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report
+              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
 
             - name: Archive Playwright E2E test report
               if: |
@@ -65,8 +68,8 @@ jobs:
               with:
                   name: e2e-test-report---pr-${{ github.event.number }}
                   path: |
-                      plugins/woocommerce/e2e/allure-results
+                      ${{ env.ALLURE_RESULTS_DIR }}
-                      plugins/woocommerce/e2e/allure-report
+                      ${{ env.ALLURE_REPORT_DIR }}
                   if-no-files-found: ignore
                   retention-days: 5
 
@@ -74,7 +77,8 @@ jobs:
         name: Runs API tests.
         runs-on: ubuntu-20.04
         env:
-            API_TEST_REPORT_DIR: ${{ github.workspace }}/api-test-report
+            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results
+            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report
         steps:
             - uses: actions/checkout@v3
 
@@ -103,7 +107,7 @@ jobs:
                     steps.run_playwright_api_tests.conclusion != 'skipped' 
                   )
               working-directory: plugins/woocommerce
-              run: pnpm exec allure generate --clean api-test-report/allure-results --output api-test-report/allure-report
+              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
             - name: Archive Playwright API test report
               if: |
                   always() &&
@@ -112,8 +116,8 @@ jobs:
               with:
                   name: api-test-report---pr-${{ github.event.number }}
                   path: |
-                      plugins/woocommerce/api-test-report/allure-results
+                      ${{ env.ALLURE_RESULTS_DIR }}
-                      plugins/woocommerce/api-test-report/allure-report
+                      ${{ env.ALLURE_REPORT_DIR }}
                   if-no-files-found: ignore
                   retention-days: 5
 

.github/workflows/smoke-test-daily.yml

@@ -68,7 +68,7 @@ jobs:
                     steps.e2e.conclusion != 'skipped' 
                   )
               working-directory: plugins/woocommerce
-              run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report
+              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
 
             - name: Archive E2E test report
               if: |
@@ -78,8 +78,8 @@ jobs:
               with:
                   name: ${{ env.E2E_ARTIFACT }}
                   path: |
-                      plugins/woocommerce/e2e/allure-results
+                      ${{ env.ALLURE_RESULTS_DIR }}
-                      plugins/woocommerce/e2e/allure-report
+                      ${{ env.ALLURE_REPORT_DIR }}
                   if-no-files-found: ignore
                   retention-days: 5
 
@@ -88,6 +88,9 @@ jobs:
         runs-on: ubuntu-20.04
         needs: [e2e-tests]
         if: always()
+        env:
+            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-results
+            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/api-test-report/allure-report
         steps:
             - uses: actions/checkout@v3
               with:
@@ -119,7 +122,7 @@ jobs:
                     steps.run_playwright_api_tests.conclusion != 'skipped' 
                   )
               working-directory: plugins/woocommerce
-              run: pnpm exec allure generate --clean api-test-report/allure-results --output api-test-report/allure-report
+              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
 
             - name: Archive API test report
               if: |
@@ -129,8 +132,8 @@ jobs:
               with:
                   name: ${{ env.API_ARTIFACT }}
                   path: |
-                      plugins/woocommerce/api-test-report/allure-results
+                      ${{ env.ALLURE_RESULTS_DIR }}
-                      plugins/woocommerce/api-test-report/allure-report
+                      ${{ env.ALLURE_REPORT_DIR }}
                   if-no-files-found: ignore
                   retention-days: 5
 
@@ -194,6 +197,8 @@ jobs:
         if: always()
         env:
             USE_WP_ENV: 1
+            ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results
+            ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report
         strategy:
             fail-fast: false
             matrix:
@@ -254,7 +259,7 @@ jobs:
                     steps.e2e.conclusion != 'skipped' 
                   )
               working-directory: plugins/woocommerce
-              run: pnpm exec allure generate --clean e2e/allure-results --output e2e/allure-report
+              run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }}
 
             - name: Archive E2E test report
               if: |
@@ -264,8 +269,8 @@ jobs:
               with:
                   name: Smoke tests with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
                   path: |
-                      plugins/woocommerce/e2e/allure-results
+                      ${{ env.ALLURE_RESULTS_DIR }}
-                      plugins/woocommerce/e2e/allure-report
+                      ${{ env.ALLURE_REPORT_DIR }}
                   if-no-files-found: ignore
                   retention-days: 5
 

plugins/woocommerce/changelog/e2e-use-e2e-pw-folder-for-outputs

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Use plugins/woocommerce/tests/e2e-pw folder for saving test outputs

plugins/woocommerce/tests/api-core-tests/playwright.config.js

@@ -34,9 +34,19 @@ const config = {
 		],
 		[
 			'allure-playwright',
-			{ outputFolder: 'api-test-report/allure-results' },
+			{
+				outputFolder:
+					process.env.ALLURE_RESULTS_DIR ??
+					'tests/api-core-tests/api-test-report/allure-results',
+			},
+		],
+		[
+			'json',
+			{
+				outputFile:
+					'tests/api-core-tests/api-test-report/test-results.json',
+			},
 		],
-		[ 'json', { outputFile: 'api-test-report/test-results.json' } ],
 	],
 	use: {
 		screenshot: 'only-on-failure',

plugins/woocommerce/tests/e2e-pw/playwright.config.js

@@ -1,9 +1,10 @@
 const { devices } = require( '@playwright/test' );
 const {
-	CI,
+	ALLURE_RESULTS_DIR,
-	E2E_MAX_FAILURES,
 	BASE_URL,
+	CI,
 	DEFAULT_TIMEOUT_OVERRIDE,
+	E2E_MAX_FAILURES,
 } = process.env;
 
 const config = {
@@ -26,17 +27,23 @@ const config = {
 				open: CI ? 'never' : 'always',
 			},
 		],
-		[ 'allure-playwright', { outputFolder: 'e2e/allure-results' } ],
+		[
-		[ 'json', { outputFile: 'e2e/test-results.json' } ],
+			'allure-playwright',
+			{
+				outputFolder:
+					ALLURE_RESULTS_DIR ?? 'tests/e2e-pw/allure-results',
+			},
+		],
+		[ 'json', { outputFile: 'tests/e2e-pw/test-results.json' } ],
 	],
 	maxFailures: E2E_MAX_FAILURES ? Number( E2E_MAX_FAILURES ) : 0,
 	use: {
-		screenshot: 'only-on-failure',
-		video: 'on-first-retry',
-		trace: 'retain-on-failure',
-		viewport: { width: 1280, height: 720 },
 		baseURL: BASE_URL ?? 'http://localhost:8086',
-		stateDir: 'e2e/storage/',
+		screenshot: 'only-on-failure',
+		stateDir: 'tests/e2e-pw/storage/',
+		trace: 'retain-on-failure',
+		video: 'on-first-retry',
+		viewport: { width: 1280, height: 720 },
 	},
 	projects: [
 		{

plugins/woocommerce/tests/e2e-pw/storage/adminState.json

@@ -1,65 +0,0 @@
-{
-  "cookies": [
-    {
-      "sameSite": "Lax",
-      "name": "wordpress_dc5025de8b60c0a511df7c07d81ead97",
-      "value": "admin%7C1660405922%7CjsNIRwmbjFh7KIGlSHy50rSs6X0L0zISppmBqgh0u0u%7C8ac4ff905331544a3cbcf0c58840ebfc3f492d4511dd17f48dbed18f5662a2c6",
-      "domain": "localhost",
-      "path": "/wp-content/plugins",
-      "expires": -1,
-      "httpOnly": true,
-      "secure": false
-    },
-    {
-      "sameSite": "Lax",
-      "name": "wordpress_dc5025de8b60c0a511df7c07d81ead97",
-      "value": "admin%7C1660405922%7CjsNIRwmbjFh7KIGlSHy50rSs6X0L0zISppmBqgh0u0u%7C8ac4ff905331544a3cbcf0c58840ebfc3f492d4511dd17f48dbed18f5662a2c6",
-      "domain": "localhost",
-      "path": "/wp-admin",
-      "expires": -1,
-      "httpOnly": true,
-      "secure": false
-    },
-    {
-      "sameSite": "Lax",
-      "name": "wordpress_test_cookie",
-      "value": "WP%20Cookie%20check",
-      "domain": "localhost",
-      "path": "/",
-      "expires": -1,
-      "httpOnly": false,
-      "secure": false
-    },
-    {
-      "sameSite": "Lax",
-      "name": "wordpress_logged_in_dc5025de8b60c0a511df7c07d81ead97",
-      "value": "admin%7C1660405922%7CjsNIRwmbjFh7KIGlSHy50rSs6X0L0zISppmBqgh0u0u%7C6de27e0d8393de7715c07e144424d90733d53e65b8665c8a7db1ad94798c369c",
-      "domain": "localhost",
-      "path": "/",
-      "expires": -1,
-      "httpOnly": true,
-      "secure": false
-    },
-    {
-      "sameSite": "Lax",
-      "name": "tk_ai",
-      "value": "woo%3AT9QFbJmpuSGWcjApaONaBtSB",
-      "domain": "localhost",
-      "path": "/",
-      "expires": -1,
-      "httpOnly": false,
-      "secure": false
-    },
-    {
-      "sameSite": "Lax",
-      "name": "wp-settings-time-1",
-      "value": "1660233126",
-      "domain": "localhost",
-      "path": "/",
-      "expires": 1691769126.937059,
-      "httpOnly": false,
-      "secure": false
-    }
-  ],
-  "origins": []
-}

plugins/woocommerce/tests/e2e-pw/storage/customerState.json

@@ -1,45 +0,0 @@
-{
-  "cookies": [
-    {
-      "sameSite": "Lax",
-      "name": "wordpress_dc5025de8b60c0a511df7c07d81ead97",
-      "value": "customer%7C1660405929%7CL0OCLRBBMubq8iKqOeQU1NqOzTWFd7ppECd1n100GKG%7Cccb6b6b9e1190e94ef751360fded6ec026c89350f9f4695e9bf900ca09d72a84",
-      "domain": "localhost",
-      "path": "/wp-content/plugins",
-      "expires": -1,
-      "httpOnly": true,
-      "secure": false
-    },
-    {
-      "sameSite": "Lax",
-      "name": "wordpress_dc5025de8b60c0a511df7c07d81ead97",
-      "value": "customer%7C1660405929%7CL0OCLRBBMubq8iKqOeQU1NqOzTWFd7ppECd1n100GKG%7Cccb6b6b9e1190e94ef751360fded6ec026c89350f9f4695e9bf900ca09d72a84",
-      "domain": "localhost",
-      "path": "/wp-admin",
-      "expires": -1,
-      "httpOnly": true,
-      "secure": false
-    },
-    {
-      "sameSite": "Lax",
-      "name": "wordpress_test_cookie",
-      "value": "WP%20Cookie%20check",
-      "domain": "localhost",
-      "path": "/",
-      "expires": -1,
-      "httpOnly": false,
-      "secure": false
-    },
-    {
-      "sameSite": "Lax",
-      "name": "wordpress_logged_in_dc5025de8b60c0a511df7c07d81ead97",
-      "value": "customer%7C1660405929%7CL0OCLRBBMubq8iKqOeQU1NqOzTWFd7ppECd1n100GKG%7C869a28fa8ef17799b6d097d84c8155c7d0dd18cd9967f7a18e3f49f41dd71226",
-      "domain": "localhost",
-      "path": "/",
-      "expires": -1,
-      "httpOnly": true,
-      "secure": false
-    }
-  ],
-  "origins": []
-}