diff --git a/.buildkite/job-version-bump.json.py b/.buildkite/job-version-bump.json.py index 61f7639878..5d5d17fef0 100755 --- a/.buildkite/job-version-bump.json.py +++ b/.buildkite/job-version-bump.json.py @@ -10,83 +10,112 @@ # # This script generates JSON for the ml-cpp version bump pipeline. # It is intended to be triggered by the centralized release-eng pipeline. -# It can be integrated into existing or new workflows and includes a plugin -# that polls artifact URLs until the expected version is available. +# +# Patch workflow: verify git push credentials (dry-run), bump version on BRANCH, +# then wait for staging and snapshot artifact JSON to publish NEW_VERSION. import contextlib import json +import os + + +WOLFI_IMAGE = "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest" +STAGING_URL = "https://artifacts-staging.elastic.co/ml-cpp/latest" +SNAPSHOT_URL = "https://storage.googleapis.com/elastic-artifacts-snapshot/ml-cpp/latest" + + +def json_watcher_plugin(url, expected_value): + return { + "elastic/json-watcher#v1.0.0": { + "url": url, + "field": ".version", + "expected_value": expected_value, + "polling_interval": "30", + } + } + + +def dra_step(label, key, depends_on, plugins): + return { + "label": label, + "key": key, + "depends_on": depends_on, + "agents": { + "image": WOLFI_IMAGE, + "cpu": "250m", + "memory": "512Mi", + "ephemeralStorage": "1Gi", + }, + "command": [ + 'echo "Waiting for DRA artifacts..."', + ], + "timeout_in_minutes": 240, + "retry": { + "automatic": [{"exit_status": "*", "limit": 2}], + "manual": {"permit_on_passed": True}, + }, + "plugins": plugins, + } def main(): - pipeline = {} - # TODO: replace the block step with version bump logic pipeline_steps = [ { - "label": "Queue a :slack: notification for the pipeline", + "label": "Verify git push credentials (dry-run)", + "key": "git-push-auth-probe", "depends_on": None, - "command": ".buildkite/pipelines/send_version_bump_notification.sh | buildkite-agent pipeline upload", "agents": { - "image": "python", + "image": WOLFI_IMAGE, + "cpu": "250m", + "memory": "512Mi", }, + "command": [ + "dev-tools/git_push_auth_probe.sh", + ], }, { - "block": "Ready to fetch for DRA artifacts?", - "prompt": ( - "Unblock when your team is ready to proceed.\n\n" - "Trigger parameters:\n" - "- NEW_VERSION: ${NEW_VERSION}\n" - "- BRANCH: ${BRANCH}\n" - "- WORKFLOW: ${WORKFLOW}\n" - ), - "key": "block-get-dra-artifacts", - "blocked_state": "running", + "label": "Queue a :slack: notification for the pipeline", + "key": "queue-slack-notify", + "depends_on": "git-push-auth-probe", + "command": ".buildkite/pipelines/send_slack_version_bump_notification.sh | buildkite-agent pipeline upload", + "agents": { + "image": "python", + }, }, { - "label": "Fetch DRA Artifacts", - "key": "fetch-dra-artifacts", - "depends_on": "block-get-dra-artifacts", + "label": "Bump version to ${NEW_VERSION}", + "key": "bump-version", + "depends_on": "queue-slack-notify", "agents": { - "image": "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest", + "image": WOLFI_IMAGE, "cpu": "250m", "memory": "512Mi", - "ephemeralStorage": "1Gi", }, "command": [ - 'echo "Starting DRA artifacts retrieval..."', - ], - "timeout_in_minutes": 240, - "retry": { - "automatic": [ - { - "exit_status": "*", - "limit": 2, - } - ], - "manual": {"permit_on_passed": True}, - }, - "plugins": [ - { - "elastic/json-watcher#v1.0.0": { - "url": "https://artifacts-staging.elastic.co/ml-cpp/latest/${BRANCH}.json", - "field": ".version", - "expected_value": "${NEW_VERSION}", - "polling_interval": "30", - } - }, - { - "elastic/json-watcher#v1.0.0": { - "url": "https://storage.googleapis.com/elastic-artifacts-snapshot/ml-cpp/latest/${BRANCH}.json", - "field": ".version", - "expected_value": "${NEW_VERSION}-SNAPSHOT", - "polling_interval": "30", - } - }, + "dev-tools/bump_version.sh", ], }, + dra_step( + label="Fetch DRA Artifacts", + key="fetch-dra-artifacts", + depends_on="bump-version", + plugins=[ + json_watcher_plugin( + f"{STAGING_URL}/${{BRANCH}}.json", + "${NEW_VERSION}", + ), + json_watcher_plugin( + f"{SNAPSHOT_URL}/${{BRANCH}}.json", + "${NEW_VERSION}-SNAPSHOT", + ), + ], + ), ] - pipeline["steps"] = pipeline_steps + pipeline = { + "steps": pipeline_steps, + } print(json.dumps(pipeline, indent=2)) diff --git a/.buildkite/pipelines/send_slack_version_bump_notification.sh b/.buildkite/pipelines/send_slack_version_bump_notification.sh new file mode 100755 index 0000000000..3e99d0c97b --- /dev/null +++ b/.buildkite/pipelines/send_slack_version_bump_notification.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Slack notifications for the ml-cpp-version-bump pipeline only (not PR builds). +# +# Optional env: +# ML_CPP_VERSION_BUMP_SLACK_CHANNEL — override channel (default #machine-learn-build) + +CHANNEL="${ML_CPP_VERSION_BUMP_SLACK_CHANNEL:-#machine-learn-build}" + +cat <> "${GITHUB_OUTPUT}" + if [[ "${DRY_IN}" == "true" ]]; then + echo "dry_run=true" >> "${GITHUB_OUTPUT}" + else + echo "dry_run=false" >> "${GITHUB_OUTPUT}" + fi + + bump: + name: Bump version (GitHub App token) + runs-on: ubuntu-latest + needs: + - prepare + steps: + - name: Mint installation token (GitHub App) + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.ML_CPP_RELEASE_GITHUB_APP_ID }} + private-key: ${{ secrets.ML_CPP_RELEASE_GITHUB_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + + - name: Checkout target branch + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + ref: ${{ needs.prepare.outputs.branch }} + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + # Match dev-tools/bump_version.sh configure_git() so commits stay consistent. + # Swap for elastic/oblt-actions/git/setup@v1 if infra prefers the shared action. + - name: Configure git (service identity) + shell: bash + run: | + git config user.name elasticsearchmachine + git config user.email 'infra-root+elasticsearchmachine@elastic.co' + + - name: Run bump script + env: + NEW_VERSION: ${{ needs.prepare.outputs.new_version }} + BRANCH: ${{ needs.prepare.outputs.branch }} + DRY_RUN: ${{ needs.prepare.outputs.dry_run }} + shell: bash + run: | + set -euo pipefail + bash dev-tools/bump_version.sh diff --git a/dev-tools/bump_version.sh b/dev-tools/bump_version.sh new file mode 100755 index 0000000000..e8cbc76960 --- /dev/null +++ b/dev-tools/bump_version.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Automated patch version bump for the release-eng pipeline. +# +# Updates elasticsearchVersion in gradle.properties to NEW_VERSION on BRANCH, +# commits, and pushes. Does not modify .backportrc.json (reserved for a future +# main / minor bump automation change). +# +# Set DRY_RUN=true to perform all steps except git push. +# +# Follows the same pattern as the Elasticsearch repo's automated +# Lucene snapshot updates (.buildkite/scripts/lucene-snapshot/). + +set -euo pipefail + +: "${NEW_VERSION:?NEW_VERSION must be set}" +: "${BRANCH:?BRANCH must be set}" +DRY_RUN="${DRY_RUN:-false}" + +GRADLE_PROPS="gradle.properties" + +if [ "$DRY_RUN" = "true" ]; then + echo "=== DRY RUN MODE — will not push ===" +fi + +git_push() { + local target_branch="$1" + if [ "$DRY_RUN" = "true" ]; then + echo " [DRY RUN] Would push $target_branch" + else + git push origin "$target_branch" + echo " Pushed $target_branch" + fi +} + +sed_inplace() { + if sed --version >/dev/null 2>&1; then + sed -i "$@" + else + sed -i '' "$@" + fi +} + +configure_git() { + git config user.name elasticsearchmachine + git config user.email 'infra-root+elasticsearchmachine@elastic.co' +} + +bump_version_on_branch() { + local target_branch="$1" + local target_version="$2" + + git checkout "$target_branch" + git pull --ff-only origin "$target_branch" + + local current_version + current_version=$(grep '^elasticsearchVersion=' "$GRADLE_PROPS" | cut -d= -f2) + if [ "$current_version" = "$target_version" ]; then + echo "Version on $target_branch is already $target_version — nothing to do" + return 0 + fi + + echo "Bumping version on $target_branch: $current_version → $target_version" + sed_inplace "s/^elasticsearchVersion=.*/elasticsearchVersion=${target_version}/" "$GRADLE_PROPS" + + if ! grep -q "^elasticsearchVersion=${target_version}$" "$GRADLE_PROPS"; then + echo "ERROR: version update verification failed on $target_branch" + grep 'elasticsearchVersion' "$GRADLE_PROPS" + exit 1 + fi + + if git diff-index --quiet HEAD --; then + echo "No changes to commit on $target_branch (file unchanged after sed)" + return 0 + fi + + configure_git + git add "$GRADLE_PROPS" + git commit -m "[ML] Bump version to ${target_version}" + git_push "$target_branch" +} + +echo "=== Patch version bump: $BRANCH → $NEW_VERSION ===" +bump_version_on_branch "$BRANCH" "$NEW_VERSION" + +if [ "$DRY_RUN" = "true" ]; then + echo "" + echo "=== DRY RUN SUMMARY ===" + echo "Branch: $BRANCH" + echo "Version: $NEW_VERSION" + echo "Recent commits:" + git log --oneline -3 +fi diff --git a/dev-tools/git_push_auth_probe.sh b/dev-tools/git_push_auth_probe.sh new file mode 100755 index 0000000000..7059d3fc2c --- /dev/null +++ b/dev-tools/git_push_auth_probe.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Verifies that the current checkout can authenticate for git push to origin, +# without updating any remote refs (uses "git push --dry-run"). +# +# Intended for the ml-cpp-version-bump Buildkite pipeline (same agent + remotes +# as dev-tools/bump_version.sh). Uses a disposable ref name under refs/heads/ci/ +# so it does not collide with release branches. +# +# Environment: +# BUILDKITE_BUILD_NUMBER — used to uniquify the probe ref (defaults to "local" +# when unset, e.g. manual runs outside Buildkite). +# GIT_REMOTE — remote name (default: origin). + +set -euo pipefail + +REMOTE="${GIT_REMOTE:-origin}" +BUILD_NUM="${BUILDKITE_BUILD_NUMBER:-local}" +PROBE_REF="refs/heads/ci/ml-cpp-bump-push-probe-${BUILD_NUM}" + +echo "=== Git push auth probe (dry-run; no remote refs updated) ===" +echo "Remote: ${REMOTE}" +echo "Local HEAD: $(git rev-parse HEAD)" +echo "Probe refspec: HEAD:${PROBE_REF}" +git remote -v + +if ! git push --dry-run "${REMOTE}" "HEAD:${PROBE_REF}"; then + echo "ERROR: git push --dry-run failed — check credentials and GitHub permissions for ${REMOTE}." >&2 + exit 1 +fi + +echo "OK: git push --dry-run succeeded for ${REMOTE}."