Add Buildkite reporting for e2e PR and daily runs (#47558)

* Add Buildkite reporting for e2e PR runs

* Set up Buildkite reporting on daily runs

---------

Co-authored-by: Jon Lane <jon.lane@automattic.com>

.github/workflows/ci.yml

@@ -116,12 +116,31 @@ jobs:
                   install: '${{ matrix.projectName }}...'
                   build: '${{ matrix.projectName }}'
 
+            - name: Get commit message
+              id: get_commit_message
+              env:
+                  HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
+                  PR_TITLE: ${{ github.event.pull_request.title }}
+              run: |
+                  if [[ "${{ github.event_name }}" == "push" ]]; then
+                    COMMIT_MESSAGE=`echo "$HEAD_COMMIT_MESSAGE" | head -1`
+                  elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
+                    COMMIT_MESSAGE="$PR_TITLE"
+                  else
+                    COMMIT_MESSAGE="${{ github.event_name }}"
+                  fi
+                  echo "COMMIT_MESSAGE=$COMMIT_MESSAGE" >> "$GITHUB_OUTPUT"
+              shell: bash
+
             - name: 'Prepare Test Environment'
               id: 'prepare-test-environment'
               if: ${{ matrix.testEnv.shouldCreate }}
               run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.testEnv.start }}'
 
             - name: 'Run tests'
+              env:
+                  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_CORE_E2E_TOKEN }}
+                  BUILDKITE_ANALYTICS_MESSAGE: ${{ steps.get_commit_message.outputs.COMMIT_MESSAGE }}
               run: 'pnpm --filter="${{ matrix.projectName }}" ${{ matrix.command }}'
 
             - name: 'Upload artifacts'

.github/workflows/smoke-test-daily.yml

@@ -119,6 +119,13 @@ jobs:
               working-directory: plugins/woocommerce
               run: pnpm exec playwright install chromium
 
+            - name: Set Buildkite message
+              id: set_buildkite_message
+              run: |
+                  BUILDKITE_MESSAGE=`echo "Daily E2E run for $(date '+%Y-%m-%d')"`
+                  echo "BUILDKITE_MESSAGE=$BUILDKITE_MESSAGE" >> "$GITHUB_OUTPUT"
+              shell: bash
+
             - name: Run E2E tests
               timeout-minutes: 90
               id: run_playwright_e2e_tests
@@ -128,6 +135,8 @@ jobs:
                   FORCE_COLOR: 1
                   ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report
                   ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results
+                  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_DAILY_E2E_TOKEN }}
+                  BUILDKITE_ANALYTICS_MESSAGE: ${{ steps.set_buildkite_message.outputs.BUILDKITE_MESSAGE }}
               working-directory: plugins/woocommerce
               run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js --shard ${{ matrix.shard.name }}
 
@@ -280,7 +289,7 @@ jobs:
               uses: ./.github/actions/tests/setup-local-test-environment
               with:
                   test-type: e2e
-                  
+
             - name: Setup plugin for the main e2e suite
               working-directory: ./plugins/woocommerce
               run: ./tests/e2e-pw/bin/install-plugin.sh
@@ -292,7 +301,7 @@ jobs:
               with:
                   report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }})
                   tests: upload-plugin.spec.js
-                  
+
             - name: Run the rest of E2E tests
               id: run-e2e-composite-action
               timeout-minutes: 90

plugins/woocommerce/changelog/dev-buildkite-on-automattic-account

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Add Buildkite test reporting (sanitized) back using production account

plugins/woocommerce/tests/e2e-pw/playwright.config.js

@@ -31,6 +31,7 @@ const reporter = [
 
 if ( process.env.CI ) {
 	reporter.push( [ 'github' ] );
+	reporter.push( [ 'buildkite-test-collector/playwright/reporter' ] );
 } else {
 	reporter.push( [
 		'html',