ci: tighten drift workflow for readability and maintainability

  - Single source of truth for the apm-server config: write it to a
    file in the CI run, then read it back (with path.data rewritten)
    when generating the recipe in the drift issue. The CI run and the
    recipe can no longer disagree about what config produced the
    captured /stats.

  - Split "Run apm-server, capture /stats" into "Write apm-server
    config" + "Run apm-server, capture /stats". The split makes the
    config the source-of-truth artifact other steps reference.

  - Replace the on-disk PID file with a bash variable plus an EXIT
    trap that cleans up apm-server even if curl fails.

  - Hoist STATS_PORT to a job-level env so the magic 15066 lives in
    one place.

  - Recipe: `export PATH="$PATH:$(go env GOPATH)/bin"` after
    `go install`, since contributors without that already in PATH
    would otherwise see "stats-to-mapping: command not found".

  - Issue title and lead paragraph now state the drifted file count
    (`stats-to-mapping drift in N file(s) (date)`) so a reader
    scanning the issue list grasps the scope without opening it.
    Workflow output renamed `drift` -> `drifted_files` (a count) and
    the conditional steps gate on != '0'.

No behavior change in what's detected or when an issue is opened.

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>

Author: carsonip

.github/workflows/stats-to-mapping-drift.yml

@@ -29,11 +29,12 @@ jobs:
     env:
       DRIFT_LABEL: stats-to-mapping-drift
       WORK: ${{ github.workspace }}/.drift
+      STATS_PORT: 15066
       # Single source of truth for the files this job tracks. Each line is
       # "<repo>/<path-within-repo>". The first slash-separated segment is
       # treated as the github.com/elastic/<repo> name; the remainder is the
       # file's path inside that repo. Sparse-checkout, the regen invocation,
-      # the drift loop, and the issue-body recipe are all derived from this.
+      # the drift loop, and the issue-body recipe all derive from this.
       UPSTREAM_FILES: |
         elasticsearch/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json
         elasticsearch/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json
@@ -55,11 +56,14 @@ jobs:
       - name: Install stats-to-mapping
         run: go install github.com/elastic/apm-tools/cmd/stats-to-mapping@latest
 
-      - name: Run apm-server, capture /stats
+      # Write the apm-server config to a file. The same file is read back later
+      # to embed in the drift issue's reproduction recipe (with path.data
+      # rewritten), so the two can never drift.
+      - name: Write apm-server config
         run: |
           set -euo pipefail
           mkdir -p "$WORK/data"
-          install -m 0600 /dev/stdin "$WORK/apm-server.yml" <<'EOF'
+          install -m 0600 /dev/stdin "$WORK/apm-server.yml" <<EOF
           apm-server:
             host: "127.0.0.1:18200"
             sampling.tail:
@@ -69,27 +73,30 @@ jobs:
                 - sample_rate: 1
           output.elasticsearch:
             hosts: ["http://127.0.0.1:9200"]
-          path.data: ${WORK}/data
+          path.data: $WORK/data
           http.enabled: true
           http.host: "127.0.0.1"
-          http.port: 15066
+          http.port: $STATS_PORT
           logging.level: warning
           EOF
-          # Substitute $WORK because the heredoc was quoted.
-          sed -i "s|\${WORK}|$WORK|" "$WORK/apm-server.yml"
-          "$WORK/apm-server" --strict.perms=false -e -c "$WORK/apm-server.yml" >"$WORK/apm-server.log" 2>&1 &
-          echo $! > "$WORK/apm-server.pid"
+
+      - name: Run apm-server, capture /stats
+        run: |
+          set -euo pipefail
+          "$WORK/apm-server" --strict.perms=false -e -c "$WORK/apm-server.yml" \
+            >"$WORK/apm-server.log" 2>&1 &
+          pid=$!
+          trap 'kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null || true' EXIT
           for attempt in $(seq 1 30); do
-            if curl -sf http://127.0.0.1:15066/stats -o "$WORK/stats.json"; then
+            if curl -sf "http://127.0.0.1:$STATS_PORT/stats" -o "$WORK/stats.json"; then
               echo "Captured /stats on attempt $attempt"
               break
             fi
             sleep 1
           done
-          kill "$(cat "$WORK/apm-server.pid")" || true
-          wait || true
           test -s "$WORK/stats.json"
-          # Sanity: TBS-enabled stats should contain sampling.tail.storage.
+          # TBS-enabled stats must include sampling.tail.storage.* — fail fast
+          # if the config or apm-server's stats shape changed.
           jq -e '.["apm-server"].sampling.tail.storage' "$WORK/stats.json" >/dev/null
 
       - name: Sparse-checkout upstream mapping files
@@ -124,7 +131,7 @@ jobs:
           set -euo pipefail
           : > "$WORK/drift.diff"
           : > "$WORK/drift.summary.md"
-          drift_found=false
+          drifted_files=0
           while IFS= read -r path; do
             [ -z "$path" ] && continue
             repo=${path%%/*}
@@ -133,7 +140,7 @@ jobs:
             if git -C "$tree" diff --quiet -- "$file"; then
               continue
             fi
-            drift_found=true
+            drifted_files=$((drifted_files + 1))
             read -r added removed _ < <(git -C "$tree" diff --numstat -- "$file")
             printf -- '- `%s/%s` — +%s / -%s\n' "$repo" "$file" "$added" "$removed" \
               >> "$WORK/drift.summary.md"
@@ -143,10 +150,10 @@ jobs:
               printf '\n```\n'
             } >> "$WORK/drift.diff"
           done <<< "$UPSTREAM_FILES"
-          echo "drift=$drift_found" >> "$GITHUB_OUTPUT"
+          echo "drifted_files=$drifted_files" >> "$GITHUB_OUTPUT"
 
       - name: Upload drift diff artifact
-        if: steps.drift.outputs.drift == 'true'
+        if: steps.drift.outputs.drifted_files != '0'
         uses: actions/upload-artifact@v4
         with:
           name: drift-diff
@@ -155,26 +162,45 @@ jobs:
             ${{ env.WORK }}/stats.json
           retention-days: 30
 
+      # If drift is detected and no open drift issue already exists, file a new
+      # issue. The issue body has three sections: a one-line summary, a list of
+      # drifted files, and a copy-pasteable local reproduction recipe.
+      #
+      # The apm-server config in the recipe is read back from the file written
+      # earlier in this job (with the CI-specific path.data rewritten) so the
+      # recipe and the CI run can never disagree about what config produced
+      # the captured /stats.
       - name: Open drift issue if none exists
-        if: steps.drift.outputs.drift == 'true'
+        if: steps.drift.outputs.drifted_files != '0'
         env:
           GH_TOKEN: ${{ github.token }}
           REPO: ${{ github.repository }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          DRIFTED_FILES: ${{ steps.drift.outputs.drifted_files }}
         run: |
           set -euo pipefail
-          existing=$(gh issue list --repo "$REPO" --label "$DRIFT_LABEL" --state open --json number --jq 'length')
+          existing=$(gh issue list --repo "$REPO" --label "$DRIFT_LABEL" \
+            --state open --json number --jq 'length')
           if [ "$existing" -gt 0 ]; then
             echo "Open drift issue already exists; not opening another"
             exit 0
           fi
+
           today=$(date -u +%Y-%m-%d)
+
+          # Build the path block for the recipe by prefixing each tracked file
+          # with /tmp/. Trailing backslashes continue the shell command.
           recipe_paths=$(printf '%s\n' "$UPSTREAM_FILES" \
-            | awk 'NF > 0 { print "    /tmp/" $0 " \\" }')
+            | awk 'NF > 0 { print "      /tmp/" $0 " \\" }')
+
+          # Reuse the exact apm-server config from the CI run, with path.data
+          # rewritten to the local /tmp location used in the recipe.
+          recipe_config=$(sed "s|$WORK/data|/tmp/apm-server-data|" "$WORK/apm-server.yml")
+
           BODY=$WORK/issue-body.md
           {
             cat <<HEADER
-          The weekly drift-check workflow detected divergence between apm-server's \`/stats\` endpoint and the upstream mapping files maintained in \`elastic/elasticsearch\`, \`elastic/beats\`, and \`elastic/integrations\`.
+          The weekly drift-check workflow detected drift in **$DRIFTED_FILES file(s)** between apm-server's \`/stats\` endpoint and the upstream mapping files in \`elastic/elasticsearch\`, \`elastic/beats\`, and \`elastic/integrations\`.
 
           Detected on $today by [workflow run]($RUN_URL). The full \`git diff\` of every drifted file is attached to that run as the \`drift-diff\` artifact, alongside the captured \`stats.json\`.
 
@@ -186,56 +212,48 @@ jobs:
 
           ## Reproduce locally
 
+          From an apm-server checkout:
+
           \`\`\`shell
-          # 1. Install the regen tool.
+          # 1. Install the regen tool. Make sure \$(go env GOPATH)/bin is on PATH.
           go install github.com/elastic/apm-tools/cmd/stats-to-mapping@latest
+          export PATH="\$PATH:\$(go env GOPATH)/bin"
 
-          # 2. Build apm-server from main and capture a TBS-enabled /stats snapshot.
+          # 2. Build apm-server and capture a TBS-enabled /stats snapshot.
           go build -o /tmp/apm-server ./x-pack/apm-server
           install -m 0600 /dev/stdin /tmp/apm-server.yml <<'CFG'
-          apm-server:
-            host: "127.0.0.1:18200"
-            sampling.tail:
-              enabled: true
-              interval: 1m
-              policies:
-                - sample_rate: 1
-          output.elasticsearch:
-            hosts: ["http://127.0.0.1:9200"]
-          path.data: /tmp/apm-server-data
-          http.enabled: true
-          http.host: "127.0.0.1"
-          http.port: 15066
-          logging.level: warning
+          $recipe_config
           CFG
           /tmp/apm-server --strict.perms=false -e -c /tmp/apm-server.yml &
+          pid=\$!
           sleep 6
-          curl -s http://127.0.0.1:15066/stats > /tmp/stats.json
-          kill %1; wait || true
+          curl -s http://127.0.0.1:$STATS_PORT/stats > /tmp/stats.json
+          kill "\$pid"; wait "\$pid" 2>/dev/null || true
 
-          # 3. Clone the three upstream repos (sparse).
+          # 3. Clone the three upstream repos.
           for r in elasticsearch beats integrations; do
             git clone --depth 1 --filter=blob:none "https://github.com/elastic/\$r" "/tmp/\$r"
           done
 
-          # 4. Run the regen.
+          # 4. Regenerate the five mapping files in place.
           stats-to-mapping \\
           $recipe_paths
               < /tmp/stats.json
 
-          # 5. Inspect the diffs and prepare upstream PRs.
+          # 5. Inspect the resulting diffs and prepare upstream PRs.
           for r in elasticsearch beats integrations; do
-            echo "=== \$r ==="; git -C "/tmp/\$r" diff
+            echo "=== \$r ==="
+            git -C "/tmp/\$r" diff
           done
           \`\`\`
 
           ## Resolution
 
-          Open a PR in each upstream repo with the diffs above, then close this issue. The workflow will not open another issue while this one stays open.
+          Open a PR in each upstream repo with the diffs above, then close this issue. The workflow will not open another drift issue while this one stays open.
           RECIPE
           } > "$BODY"
           gh issue create --repo "$REPO" \
-            --title "Monitoring metric mappings drifted from /stats ($today)" \
+            --title "stats-to-mapping drift in $DRIFTED_FILES file(s) ($today)" \
             --label bug \
             --label "$DRIFT_LABEL" \
             --body-file "$BODY"