From ad19ff51a3f6f79aab1c353851e4e331a4a09c5d Mon Sep 17 00:00:00 2001
From: Jonathan Lane <lanej0@users.noreply.github.com>
Date: Thu, 16 May 2024 11:31:21 -0700
Subject: [PATCH] Add Buildkite reporting for e2e PR and daily runs (#47558)

* Add Buildkite reporting for e2e PR runs

* Set up Buildkite reporting on daily runs

---------

Co-authored-by: Jon Lane <jon.lane@automattic.com>
---
 .github/workflows/ci.yml                      | 19 +++++++++++++++++++
 .github/workflows/smoke-test-daily.yml        | 13 +++++++++++--
 .../dev-buildkite-on-automattic-account       |  4 ++++
 .../tests/e2e-pw/playwright.config.js         |  1 +
 4 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 plugins/woocommerce/changelog/dev-buildkite-on-automattic-account

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 43d8a56311c..2a365467c84 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -116,12 +116,31 @@ jobs:
                   install: '${{ matrix.projectName }}...'
                   build: '${{ matrix.projectName }}'
 
+            - name: Get commit message
+              id: get_commit_message
+              env:
+                  HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
+                  PR_TITLE: ${{ github.event.pull_request.title }}
+              run: |
+                  if [[ "${{ github.event_name }}" == "push" ]]; then
+                    COMMIT_MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
+                  elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
+                    COMMIT_MESSAGE="$PR_TITLE"
+                  else
+                    COMMIT_MESSAGE="${{ github.event_name }}"
+                  fi
+                  echo "COMMIT_MESSAGE=$COMMIT_MESSAGE" >> "$GITHUB_OUTPUT"
+              shell: bash
+
             - name: 'Prepare Test Environment'
               id: 'prepare-test-environment'
               if: ${{ matrix.testEnv.shouldCreate }}
               run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
 
             - name: 'Run tests'
+              env:
+                  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }}
+                  BUILDKITE_ANALYTICS_MESSAGE: ${{ steps.get_commit_message.outputs.COMMIT_MESSAGE }}
               run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
 
             - name: 'Upload artifacts'
diff --git a/.github/workflows/smoke-test-daily.yml b/.github/workflows/smoke-test-daily.yml
index 93f25860e97..6c71dec7c00 100644
--- a/.github/workflows/smoke-test-daily.yml
+++ b/.github/workflows/smoke-test-daily.yml
@@ -119,6 +119,13 @@ jobs:
               working-directory: plugins/woocommerce
               run: pnpm exec playwright install chromium
 
+            - name: Set Buildkite message
+              id: set_buildkite_message
+              run: |
+                  BUILDKITE_MESSAGE=`echo "Daily E2E run for $(date '+%Y-%m-%d')"`
+                  echo "BUILDKITE_MESSAGE=$BUILDKITE_MESSAGE" >> "$GITHUB_OUTPUT"
+              shell: bash
+
             - name: Run E2E tests
               timeout-minutes: 90
               id: run_playwright_e2e_tests
@@ -128,6 +135,8 @@ jobs:
                   FORCE_COLOR: 1
                   ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
                   ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
+                  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_DAILY_E2E_TOKEN }}
+                  BUILDKITE_ANALYTICS_MESSAGE: ${{ steps.set_buildkite_message.outputs.BUILDKITE_MESSAGE }}
               working-directory: plugins/woocommerce
               run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js --shard ${{ matrix.shard.name }}
 
@@ -280,7 +289,7 @@ jobs:
               uses: ./.github/actions/tests/setup-local-test-environment
               with:
                   test-type: e2e
-                  
+
             - name: Setup plugin for the main e2e suite
               working-directory: ./plugins/woocommerce
               run: ./tests/e2e-pw/bin/install-plugin.sh
@@ -292,7 +301,7 @@ jobs:
               with:
                   report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
                   tests: upload-plugin.spec.js
-                  
+
             - name: Run the rest of E2E tests
               id: run-e2e-composite-action
               timeout-minutes: 90
diff --git a/plugins/woocommerce/changelog/dev-buildkite-on-automattic-account b/plugins/woocommerce/changelog/dev-buildkite-on-automattic-account
new file mode 100644
index 00000000000..0fc282f8c8c
--- /dev/null
+++ b/plugins/woocommerce/changelog/dev-buildkite-on-automattic-account
@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Add Buildkite test reporting (sanitized) back using production account
diff --git a/plugins/woocommerce/tests/e2e-pw/playwright.config.js b/plugins/woocommerce/tests/e2e-pw/playwright.config.js
index 36b6b128f90..743dba84310 100644
--- a/plugins/woocommerce/tests/e2e-pw/playwright.config.js
+++ b/plugins/woocommerce/tests/e2e-pw/playwright.config.js
@@ -31,6 +31,7 @@ const reporter = [
 
 if ( process.env.CI ) {
 	reporter.push( [ 'github' ] );
+	reporter.push( [ 'buildkite-test-collector/playwright/reporter' ] );
 } else {
 	reporter.push( [
 		'html',