[ML] Automate patch version bump in CI pipeline

.buildkite/job-version-bump.json.py

@@ -10,83 +10,112 @@
 #
 # This script generates JSON for the ml-cpp version bump pipeline.
 # It is intended to be triggered by the centralized release-eng pipeline.
-# It can be integrated into existing or new workflows and includes a plugin
-# that polls artifact URLs until the expected version is available.
+#
+# Patch workflow: verify git push credentials (dry-run), bump version on BRANCH,
+# then wait for staging and snapshot artifact JSON to publish NEW_VERSION.
 
 
 import contextlib
 import json
+import os
+
+
+WOLFI_IMAGE = "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest"
+STAGING_URL = "https://artifacts-staging.elastic.co/ml-cpp/latest"
+SNAPSHOT_URL = "https://storage.googleapis.com/elastic-artifacts-snapshot/ml-cpp/latest"
+
+
+def json_watcher_plugin(url, expected_value):
+    return {
+        "elastic/json-watcher#v1.0.0": {
+            "url": url,
+            "field": ".version",
+            "expected_value": expected_value,
+            "polling_interval": "30",
+        }
+    }
+
+
+def dra_step(label, key, depends_on, plugins):
+    return {
+        "label": label,
+        "key": key,
+        "depends_on": depends_on,
+        "agents": {
+            "image": WOLFI_IMAGE,
+            "cpu": "250m",
+            "memory": "512Mi",
+            "ephemeralStorage": "1Gi",
+        },
+        "command": [
+            'echo "Waiting for DRA artifacts..."',
+        ],
+        "timeout_in_minutes": 240,
+        "retry": {
+            "automatic": [{"exit_status": "*", "limit": 2}],
+            "manual": {"permit_on_passed": True},
+        },
+        "plugins": plugins,
+    }
 
 
 def main():
-    pipeline = {}
-    # TODO: replace the block step with version bump logic
     pipeline_steps = [
         {
-            "label": "Queue a :slack: notification for the pipeline",
+            "label": "Verify git push credentials (dry-run)",
+            "key": "git-push-auth-probe",
             "depends_on": None,
-            "command": ".buildkite/pipelines/send_version_bump_notification.sh | buildkite-agent pipeline upload",
             "agents": {
-                "image": "python",
+                "image": WOLFI_IMAGE,
+                "cpu": "250m",
+                "memory": "512Mi",
             },
+            "command": [
+                "dev-tools/git_push_auth_probe.sh",
+            ],
         },
         {
-            "block": "Ready to fetch for DRA artifacts?",
-            "prompt": (
-                "Unblock when your team is ready to proceed.\n\n"
-                "Trigger parameters:\n"
-                "- NEW_VERSION: ${NEW_VERSION}\n"
-                "- BRANCH: ${BRANCH}\n"
-                "- WORKFLOW: ${WORKFLOW}\n"
-            ),
-            "key": "block-get-dra-artifacts",
-            "blocked_state": "running",
+            "label": "Queue a :slack: notification for the pipeline",
+            "key": "queue-slack-notify",
+            "depends_on": "git-push-auth-probe",
+            "command": ".buildkite/pipelines/send_slack_version_bump_notification.sh | buildkite-agent pipeline upload",
+            "agents": {
+                "image": "python",
+            },
         },
         {
-            "label": "Fetch DRA Artifacts",
-            "key": "fetch-dra-artifacts",
-            "depends_on": "block-get-dra-artifacts",
+            "label": "Bump version to ${NEW_VERSION}",
+            "key": "bump-version",
+            "depends_on": "queue-slack-notify",
             "agents": {
-                "image": "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest",
+                "image": WOLFI_IMAGE,
                 "cpu": "250m",
                 "memory": "512Mi",
-                "ephemeralStorage": "1Gi",
             },
             "command": [
-                'echo "Starting DRA artifacts retrieval..."',
-            ],
-            "timeout_in_minutes": 240,
-            "retry": {
-                "automatic": [
-                    {
-                        "exit_status": "*",
-                        "limit": 2,
-                    }
-                ],
-                "manual": {"permit_on_passed": True},
-            },
-            "plugins": [
-                {
-                    "elastic/json-watcher#v1.0.0": {
-                        "url": "https://artifacts-staging.elastic.co/ml-cpp/latest/${BRANCH}.json",
-                        "field": ".version",
-                        "expected_value": "${NEW_VERSION}",
-                        "polling_interval": "30",
-                    }
-                },
-                {
-                    "elastic/json-watcher#v1.0.0": {
-                        "url": "https://storage.googleapis.com/elastic-artifacts-snapshot/ml-cpp/latest/${BRANCH}.json",
-                        "field": ".version",
-                        "expected_value": "${NEW_VERSION}-SNAPSHOT",
-                        "polling_interval": "30",
-                    }
-                },
+                "dev-tools/bump_version.sh",
             ],
         },
+        dra_step(
+            label="Fetch DRA Artifacts",
+            key="fetch-dra-artifacts",
+            depends_on="bump-version",
+            plugins=[
+                json_watcher_plugin(
+                    f"{STAGING_URL}/${{BRANCH}}.json",
+                    "${NEW_VERSION}",
+                ),
+                json_watcher_plugin(
+                    f"{SNAPSHOT_URL}/${{BRANCH}}.json",
+                    "${NEW_VERSION}-SNAPSHOT",
+                ),
+            ],
+        ),
     ]
 
-    pipeline["steps"] = pipeline_steps
+    pipeline = {
+        "steps": pipeline_steps,
+    }
 
     print(json.dumps(pipeline, indent=2))
 

.buildkite/pipelines/send_slack_version_bump_notification.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+# Slack notifications for the ml-cpp-version-bump pipeline only (not PR builds).
+#
+# Optional env:
+#   ML_CPP_VERSION_BUMP_SLACK_CHANNEL — override channel (default #machine-learn-build)
+
+CHANNEL="${ML_CPP_VERSION_BUMP_SLACK_CHANNEL:-#machine-learn-build}"
+
+cat <<EOL
+steps:
+  - label: "Schedule :slack: notification (version bump)"
+    command: "echo schedule :slack: notification"
+notify:
+  - slack:
+      channels:
+        - "${CHANNEL}"
+      message: |
+        **Version bump pipeline**
+        Build message: \${BUILDKITE_MESSAGE:-"(none)"}
+        Branch: \${BUILDKITE_BRANCH}
+        User: \${BUILDKITE_BUILD_CREATOR}
+        NEW_VERSION: \${NEW_VERSION:-"(unset)"}
+        BRANCH (param): \${BRANCH:-"(unset)"}
+        DRY_RUN: \${DRY_RUN:-"(unset)"}
+        Pipeline: \${BUILDKITE_BUILD_URL}
+        Build: \${BUILDKITE_BUILD_NUMBER}
+    if: build.pull_request.id == null
+EOL

.github/workflows/run-patch-release.yml

@@ -0,0 +1,115 @@
+# Skeleton: patch-oriented version bump via GitHub Actions (APM-style entry point).
+# Buildkite (or humans) can trigger this with `gh workflow run` once secrets exist.
+#
+# Required repository secrets (names are suggestions — align with infra / reuse an
+# existing org GitHub App after review):
+#   ML_CPP_RELEASE_GITHUB_APP_ID
+#   ML_CPP_RELEASE_GITHUB_APP_PRIVATE_KEY   (PEM for the app; multiline secret)
+#
+# Optional (Slack — mirror elastic/apm-server when ready):
+#   SLACK_BOT_TOKEN
+#
+# Install the GitHub App on elastic/ml-cpp with contents:write + pull-requests:write
+# (same class of permissions as elastic/apm-server run-patch-release.yml).
+
+name: ml-cpp run patch release
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "Git branch to bump (e.g. 9.5 or feature/foo — must exist on origin)"
+        required: true
+        type: string
+      new_version:
+        description: "Target elasticsearchVersion in gradle.properties (x.y.z)"
+        required: true
+        type: string
+      dry_run:
+        description: "If true, bump_version.sh runs with DRY_RUN=true (no git push)"
+        required: false
+        type: boolean
+        default: true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.run_id }}
+
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    name: Prepare (validate inputs)
+    runs-on: ubuntu-latest
+    outputs:
+      branch: ${{ steps.meta.outputs.branch }}
+      new_version: ${{ steps.meta.outputs.new_version }}
+      dry_run: ${{ steps.meta.outputs.dry_run }}
+    steps:
+      - name: Validate and emit outputs
+        id: meta
+        shell: bash
+        env:
+          BRANCH_IN: ${{ inputs.branch }}
+          VERSION_IN: ${{ inputs.new_version }}
+          DRY_IN: ${{ inputs.dry_run }}
+        run: |
+          set -euo pipefail
+          if [[ -z "${BRANCH_IN// }" ]]; then
+            echo "::error::branch must not be empty"
+            exit 1
+          fi
+          if ! echo "${VERSION_IN}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
+            echo "::error::new_version must look like major.minor.patch (digits only)"
+            exit 1
+          fi
+          {
+            echo "branch=${BRANCH_IN}"
+            echo "new_version=${VERSION_IN}"
+          } >> "${GITHUB_OUTPUT}"
+          if [[ "${DRY_IN}" == "true" ]]; then
+            echo "dry_run=true" >> "${GITHUB_OUTPUT}"
+          else
+            echo "dry_run=false" >> "${GITHUB_OUTPUT}"
+          fi
+
+  bump:
+    name: Bump version (GitHub App token)
+    runs-on: ubuntu-latest
+    needs:
+      - prepare
+    steps:
+      - name: Mint installation token (GitHub App)
+        id: app-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.ML_CPP_RELEASE_GITHUB_APP_ID }}
+          private-key: ${{ secrets.ML_CPP_RELEASE_GITHUB_APP_PRIVATE_KEY }}
+          permission-contents: write
+          permission-pull-requests: write
+
+      - name: Checkout target branch
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository }}
+          ref: ${{ needs.prepare.outputs.branch }}
+          fetch-depth: 0
+          token: ${{ steps.app-token.outputs.token }}
+
+      # Match dev-tools/bump_version.sh configure_git() so commits stay consistent.
+      # Swap for elastic/oblt-actions/git/setup@v1 if infra prefers the shared action.
+      - name: Configure git (service identity)
+        shell: bash
+        run: |
+          git config user.name elasticsearchmachine
+          git config user.email 'infra-root+elasticsearchmachine@elastic.co'
+
+      - name: Run bump script
+        env:
+          NEW_VERSION: ${{ needs.prepare.outputs.new_version }}
+          BRANCH: ${{ needs.prepare.outputs.branch }}
+          DRY_RUN: ${{ needs.prepare.outputs.dry_run }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          bash dev-tools/bump_version.sh