From 0c5b4ea96f2379c8c1a40942cad0edc0071749ca Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 16 Apr 2026 14:27:16 +1200 Subject: [PATCH 01/12] Add serverless MKI testing for ml-cpp PR builds Add pipeline and scripts to build a serverless ES Docker image with custom ml-cpp artifacts and run E2E tests against MKI QA. Triggered via PR comment (`buildkite run_serverless_tests`) or label (`ci:run-serverless-tests`). Builds Linux x86_64 ml-cpp, sets up a local Ivy repo, clones elasticsearch-serverless, builds the Docker image with -Dbuild.ml_cpp.repo override, pushes to the CI registry, and triggers the E2E QA pipeline. Relates elastic/ml-team#1243 Made-with: Cursor --- .buildkite/ml_pipeline/config.py | 16 ++- .buildkite/pipeline.json.py | 3 + .../pipelines/run_serverless_tests.yml.sh | 65 +++++++++++ .buildkite/pull-requests.json | 2 +- .../scripts/steps/build_serverless_docker.sh | 104 ++++++++++++++++++ 5 files changed, 185 insertions(+), 5 deletions(-) create mode 100755 .buildkite/pipelines/run_serverless_tests.yml.sh create mode 100755 .buildkite/scripts/steps/build_serverless_docker.sh diff --git a/.buildkite/ml_pipeline/config.py b/.buildkite/ml_pipeline/config.py index 7abb4a5371..df9fc0dd0d 100644 --- a/.buildkite/ml_pipeline/config.py +++ b/.buildkite/ml_pipeline/config.py @@ -19,6 +19,7 @@ class Config: build_x86_64: str = "" run_qa_tests: bool = False run_pytorch_tests: bool = False + run_serverless_tests: bool = False action: str = "build" def parse_comment(self): @@ -37,7 +38,8 @@ def parse_comment(self): self.action = os.environ["GITHUB_PR_COMMENT_VAR_ACTION"] self.run_qa_tests = self.action == "run_qa_tests" self.run_pytorch_tests = self.action == "run_pytorch_tests" - if self.run_pytorch_tests or self.run_qa_tests: + self.run_serverless_tests = self.action == "run_serverless_tests" + if self.run_pytorch_tests or self.run_qa_tests or self.run_serverless_tests: self.action = "build" # If the ACTION is set to "run_qa_tests" then set some optional variables governing the ES branch to build, the @@ -64,7 +66,7 @@ def parse_comment(self): self.build_aarch64 = "--build-aarch64" elif each == "x86_64": self.build_x86_64 = "--build-x86_64" - elif self.run_qa_tests or self.run_pytorch_tests: + elif self.run_qa_tests or self.run_pytorch_tests or self.run_serverless_tests: self.build_x86_64 = "--build-x86_64" else: self.build_aarch64 = "--build-aarch64" @@ -83,7 +85,7 @@ def parse_comment(self): self.build_macos = True elif each == "linux": self.build_linux = True - elif self.run_qa_tests or self.run_pytorch_tests: + elif self.run_qa_tests or self.run_pytorch_tests or self.run_serverless_tests: self.build_linux = True else: self.build_windows = True @@ -100,11 +102,13 @@ def parse_comment(self): self.run_qa_tests = True if "ci:run-pytorch-tests" in labels: self.run_pytorch_tests = True + if "ci:run-serverless-tests" in labels: + self.run_serverless_tests = True def parse_label(self): """ Parse labels set on GitHub PR comments.""" - build_labels = ['ci:build-linux','ci:build-macos','ci:build-windows','ci:run-qa-tests','ci:run-pytorch-tests','ci:build-aarch64','ci:build-x86_64'] + build_labels = ['ci:build-linux','ci:build-macos','ci:build-windows','ci:run-qa-tests','ci:run-pytorch-tests','ci:run-serverless-tests','ci:build-aarch64','ci:build-x86_64'] all_labels = [x.strip().lower() for x in os.environ["GITHUB_PR_LABELS"].split(",")] ci_labels = [label for label in all_labels if re.search("|".join(build_labels), label)] if not ci_labels: @@ -137,6 +141,10 @@ def parse_label(self): self.build_macos = True self.build_linux = True self.run_pytorch_tests = True + if "ci:run-serverless-tests" == label: + self.build_linux = True + self.build_x86_64 = "--build-x86_64" + self.run_serverless_tests = True if self.build_aarch64 == "" and self.build_x86_64 == "": self.build_aarch64 = "--build-aarch64" self.build_x86_64 = "--build-x86_64" diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 0ae5776853..08a802d567 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -75,6 +75,9 @@ def main(): if config.run_pytorch_tests: pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline", ".buildkite/pipelines/run_pytorch_tests.yml.sh")) + if config.run_serverless_tests: + pipeline_steps.append(pipeline_steps.generate_step("Upload serverless tests runner pipeline", + ".buildkite/pipelines/run_serverless_tests.yml.sh")) if config.build_aarch64: pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline", ".buildkite/pipelines/run_es_tests_aarch64.yml.sh")) diff --git a/.buildkite/pipelines/run_serverless_tests.yml.sh b/.buildkite/pipelines/run_serverless_tests.yml.sh new file mode 100755 index 0000000000..519529383b --- /dev/null +++ b/.buildkite/pipelines/run_serverless_tests.yml.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +# Pipeline: build a custom ml-cpp, package it into a serverless ES Docker +# image, push the image, and trigger E2E tests against MKI QA. +# +# Depends on the linux x86_64 build step having already produced +# build/distributions/ml-cpp-*-linux-x86_64.zip as a Buildkite artifact. + +SAFE_MESSAGE=$(printf '%s' "${BUILDKITE_MESSAGE}" | head -1 | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g') +PR_NUM="${BUILDKITE_PULL_REQUEST}" +if [ -z "${PR_NUM}" ] || [ "${PR_NUM}" = "false" ]; then + PR_NUM="manual" +fi +IMAGE_TAG="ml-cpp-pr-${PR_NUM}-${BUILDKITE_BUILD_NUMBER}" + +cat <build|debug|run_qa_tests|run_pytorch_tests)(=(?(?:[^ ]+)))? *(?: for ES_BRANCH=(?([.0-9a-zA-Z]+)))? *(?:with STACK_VERSION=(?([.0-9]+)))? *(?: *on *(?(?:[ ,]*(?:windows|linux|mac(os)?))+))?) *(?(?:[, ]*aarch64|x86_64)+)?$", + "trigger_comment_regex": "^(?:(?:buildkite +)(?build|debug|run_qa_tests|run_pytorch_tests|run_serverless_tests)(=(?(?:[^ ]+)))? *(?: for ES_BRANCH=(?([.0-9a-zA-Z]+)))? *(?:with STACK_VERSION=(?([.0-9]+)))? *(?: *on *(?(?:[ ,]*(?:windows|linux|mac(os)?))+))?) *(?(?:[, ]*aarch64|x86_64)+)?$", "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", "skip_ci_labels": ["skip-ci", "jenkins-ci", ">test-mute", ">docs"], "skip_target_branches": ["6.8", "7.11", "7.12"], diff --git a/.buildkite/scripts/steps/build_serverless_docker.sh b/.buildkite/scripts/steps/build_serverless_docker.sh new file mode 100755 index 0000000000..7c53b20509 --- /dev/null +++ b/.buildkite/scripts/steps/build_serverless_docker.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +# Builds a serverless Elasticsearch Docker image that incorporates custom +# ml-cpp artifacts from the current PR build. +# +# Prerequisites: +# - ml-cpp build artifacts in build/distributions/ml-cpp-*-linux-x86_64.zip +# (downloaded from a prior Buildkite step) +# +# Environment: +# IMAGE_TAG - Docker image tag (required) +# ES_SERVERLESS_BRANCH - elasticsearch-serverless branch to build from (default: main) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +ES_SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH:-main}" +DOCKER_REGISTRY="docker.elastic.co/elasticsearch-ci/elasticsearch-serverless" + +VERSION=$(grep '^elasticsearchVersion' "${REPO_ROOT}/gradle.properties" | awk -F= '{ print $2 }' | xargs echo) +VERSION="${VERSION}-SNAPSHOT" + +echo "--- Setting up local Ivy repo with custom ml-cpp ${VERSION}" +IVY_REPO="$(pwd)/ivy-repo" +IVY_ML_DIR="${IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${VERSION}" +mkdir -p "$IVY_ML_DIR" + +cp "build/distributions/ml-cpp-${VERSION}-linux-x86_64.zip" \ + "${IVY_ML_DIR}/ml-cpp-${VERSION}.zip" +cp "build/distributions/ml-cpp-${VERSION}-linux-x86_64.zip" \ + "${IVY_ML_DIR}/ml-cpp-${VERSION}-nodeps.zip" +cp "${REPO_ROOT}/dev-tools/minimal.zip" \ + "${IVY_ML_DIR}/ml-cpp-${VERSION}-deps.zip" + +IVY_REPO_URL="file://${IVY_REPO}" +echo "Ivy repo URL: ${IVY_REPO_URL}" + +echo "--- Obtaining GitHub token for elasticsearch-serverless" +set +x +# ES_SERVERLESS_GITHUB_TOKEN can be provided as a build env var when the +# pipeline's default VAULT_GITHUB_TOKEN lacks access to elasticsearch-serverless. +# Long-term, the ml-cpp pipeline's Vault role should be granted read access to +# a GitHub App token that covers elasticsearch-serverless. +ES_SERVERLESS_TOKEN="${ES_SERVERLESS_GITHUB_TOKEN:-${VAULT_GITHUB_TOKEN:-}}" +if [ -z "${ES_SERVERLESS_TOKEN}" ]; then + echo "ERROR: Could not obtain a GitHub token with access to elasticsearch-serverless." + echo "Set ES_SERVERLESS_GITHUB_TOKEN as a build environment variable." + exit 1 +fi +set -x + +echo "--- Cloning elasticsearch-serverless (branch: ${ES_SERVERLESS_BRANCH})" +cd .. +rm -rf elasticsearch-serverless +set +x +git clone --depth=1 -b "${ES_SERVERLESS_BRANCH}" \ + "https://x-access-token:${ES_SERVERLESS_TOKEN}@github.com/elastic/elasticsearch-serverless.git" +set -x +cd elasticsearch-serverless + +echo "--- Initializing elasticsearch submodule" +set +x +git config url."https://x-access-token:${ES_SERVERLESS_TOKEN}@github.com/".insteadOf "git@github.com:" +set -x +git submodule update --init --depth=1 + +echo "--- Building serverless Docker image with custom ml-cpp" +# The serverless build requires a license key for release builds +LICENSE_KEY=$(mktemp -d)/license.key +vault read -field pubkey \ + secret/ci/elastic-elasticsearch-serverless/migrated/es-license \ + | base64 --decode > "$LICENSE_KEY" + +./gradlew --console=plain --parallel \ + -Dbuild.snapshot=false \ + "-Dlicense.key=${LICENSE_KEY}" \ + "-Dbuild.ml_cpp.repo=${IVY_REPO_URL}" \ + buildDockerImage + +echo "--- Tagging and pushing Docker image" +FULL_TAG="${DOCKER_REGISTRY}:${IMAGE_TAG}" +docker tag elasticsearch-serverless:x86_64 "${FULL_TAG}" + +set +x +DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch-serverless/prod_docker_registry_credentials)" +DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch-serverless/prod_docker_registry_credentials)" +echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin docker.elastic.co +set -x + +docker push "${FULL_TAG}" +echo "Pushed ${FULL_TAG}" + +# Store the image tag as metadata for downstream steps +buildkite-agent meta-data set "serverless-image-tag" "${IMAGE_TAG}" +buildkite-agent meta-data set "serverless-image" "${FULL_TAG}" From 0b5d1dbac355bb35efb5b35d599a48d4a04e4540 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 17 Apr 2026 11:20:10 +1200 Subject: [PATCH 02/12] Restructure serverless testing to avoid cross-repo clone Instead of cloning elasticsearch-serverless from the ml-cpp pipeline (which requires cross-repo GitHub token access), upload the custom ml-cpp artifacts to an S3 staging path under the existing prelert-artifacts bucket, then trigger the serverless validation pipeline which runs in its own context with full permissions. The serverless pipeline picks up the custom artifacts via the ML_CPP_REPO_OVERRIDE env var, which is passed through as -Dbuild.ml_cpp.repo to the Gradle build. This requires a small change in elasticsearch-serverless (documented in docs/serverless-integration/). Made-with: Cursor --- .buildkite/ml_pipeline/config.py | 21 ++- .buildkite/pipeline.json.py | 11 +- .../pipelines/deploy_serverless_qa.yml.sh | 101 +++++++++++++ .../pipelines/run_serverless_tests.yml.sh | 135 +++++++++++++----- .buildkite/pull-requests.json | 2 +- .../scripts/steps/build_serverless_docker.sh | 104 -------------- 6 files changed, 226 insertions(+), 148 deletions(-) create mode 100755 .buildkite/pipelines/deploy_serverless_qa.yml.sh delete mode 100755 .buildkite/scripts/steps/build_serverless_docker.sh diff --git a/.buildkite/ml_pipeline/config.py b/.buildkite/ml_pipeline/config.py index df9fc0dd0d..184ecc93d1 100644 --- a/.buildkite/ml_pipeline/config.py +++ b/.buildkite/ml_pipeline/config.py @@ -20,6 +20,7 @@ class Config: run_qa_tests: bool = False run_pytorch_tests: bool = False run_serverless_tests: bool = False + deploy_serverless_qa: bool = False action: str = "build" def parse_comment(self): @@ -39,7 +40,8 @@ def parse_comment(self): self.run_qa_tests = self.action == "run_qa_tests" self.run_pytorch_tests = self.action == "run_pytorch_tests" self.run_serverless_tests = self.action == "run_serverless_tests" - if self.run_pytorch_tests or self.run_qa_tests or self.run_serverless_tests: + self.deploy_serverless_qa = self.action == "deploy_serverless_qa" + if self.run_pytorch_tests or self.run_qa_tests or self.run_serverless_tests or self.deploy_serverless_qa: self.action = "build" # If the ACTION is set to "run_qa_tests" then set some optional variables governing the ES branch to build, the @@ -66,7 +68,10 @@ def parse_comment(self): self.build_aarch64 = "--build-aarch64" elif each == "x86_64": self.build_x86_64 = "--build-x86_64" - elif self.run_qa_tests or self.run_pytorch_tests or self.run_serverless_tests: + elif self.run_qa_tests or self.run_pytorch_tests: + self.build_x86_64 = "--build-x86_64" + elif self.run_serverless_tests or self.deploy_serverless_qa: + self.build_aarch64 = "--build-aarch64" self.build_x86_64 = "--build-x86_64" else: self.build_aarch64 = "--build-aarch64" @@ -85,7 +90,7 @@ def parse_comment(self): self.build_macos = True elif each == "linux": self.build_linux = True - elif self.run_qa_tests or self.run_pytorch_tests or self.run_serverless_tests: + elif self.run_qa_tests or self.run_pytorch_tests or self.run_serverless_tests or self.deploy_serverless_qa: self.build_linux = True else: self.build_windows = True @@ -104,11 +109,13 @@ def parse_comment(self): self.run_pytorch_tests = True if "ci:run-serverless-tests" in labels: self.run_serverless_tests = True + if "ci:deploy-serverless-qa" in labels: + self.deploy_serverless_qa = True def parse_label(self): """ Parse labels set on GitHub PR comments.""" - build_labels = ['ci:build-linux','ci:build-macos','ci:build-windows','ci:run-qa-tests','ci:run-pytorch-tests','ci:run-serverless-tests','ci:build-aarch64','ci:build-x86_64'] + build_labels = ['ci:build-linux','ci:build-macos','ci:build-windows','ci:run-qa-tests','ci:run-pytorch-tests','ci:run-serverless-tests','ci:deploy-serverless-qa','ci:build-aarch64','ci:build-x86_64'] all_labels = [x.strip().lower() for x in os.environ["GITHUB_PR_LABELS"].split(",")] ci_labels = [label for label in all_labels if re.search("|".join(build_labels), label)] if not ci_labels: @@ -143,8 +150,14 @@ def parse_label(self): self.run_pytorch_tests = True if "ci:run-serverless-tests" == label: self.build_linux = True + self.build_aarch64 = "--build-aarch64" self.build_x86_64 = "--build-x86_64" self.run_serverless_tests = True + if "ci:deploy-serverless-qa" == label: + self.build_linux = True + self.build_aarch64 = "--build-aarch64" + self.build_x86_64 = "--build-x86_64" + self.deploy_serverless_qa = True if self.build_aarch64 == "" and self.build_x86_64 == "": self.build_aarch64 = "--build-aarch64" self.build_x86_64 = "--build-x86_64" diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 08a802d567..0a0a9406cb 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -75,13 +75,18 @@ def main(): if config.run_pytorch_tests: pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline", ".buildkite/pipelines/run_pytorch_tests.yml.sh")) - if config.run_serverless_tests: - pipeline_steps.append(pipeline_steps.generate_step("Upload serverless tests runner pipeline", - ".buildkite/pipelines/run_serverless_tests.yml.sh")) if config.build_aarch64: pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline", ".buildkite/pipelines/run_es_tests_aarch64.yml.sh")) + # Serverless tests require both x86_64 and aarch64 Linux builds. + if config.run_serverless_tests: + pipeline_steps.append(pipeline_steps.generate_step("Upload serverless tests runner pipeline", + ".buildkite/pipelines/run_serverless_tests.yml.sh")) + if config.deploy_serverless_qa: + pipeline_steps.append(pipeline_steps.generate_step("Upload serverless QA deploy pipeline", + ".buildkite/pipelines/deploy_serverless_qa.yml.sh")) + # Check for build timing regressions against nightly baseline pipeline_steps.append(pipeline_steps.generate_step("Check build timing regressions", ".buildkite/pipelines/check_build_regression.yml.sh", diff --git a/.buildkite/pipelines/deploy_serverless_qa.yml.sh b/.buildkite/pipelines/deploy_serverless_qa.yml.sh new file mode 100755 index 0000000000..28872f4cbe --- /dev/null +++ b/.buildkite/pipelines/deploy_serverless_qa.yml.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +# Pipeline: build a serverless Docker image with custom ml-cpp and deploy it +# to the QA environment for interactive use. Unlike run_serverless_tests.yml.sh, +# this does NOT run E2E tests -- it just gets the environment running so the +# developer can interact with it (deploy models, run queries, kubectl, etc.). +# +# The deployment stays up for 1 hour by default. Set KEEP_DEPLOYMENT=true +# (via the Buildkite UI) to keep it longer. The build annotations will +# contain the URL and encrypted credentials for accessing the deployment. + +SAFE_MESSAGE=$(printf '%s' "${BUILDKITE_MESSAGE}" | head -1 | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g') +PR_NUM="${BUILDKITE_PULL_REQUEST}" +if [ -z "${PR_NUM}" ] || [ "${PR_NUM}" = "false" ]; then + PR_NUM="manual" +fi + +# Extract PR metadata once for reuse. +PR_AUTHOR_FORK="$(expr "${BUILDKITE_BRANCH:-}" : '\(.*\):.*' 2>/dev/null || true)" +PR_SOURCE="$(expr "${BUILDKITE_BRANCH:-}" : '.*:\(.*\)' 2>/dev/null || true)" +PR_TARGET="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" + +# --- Resolve elasticsearch-serverless branch --- +SERVERLESS_BRANCH="main" + +check_serverless_branch() { + local repo="$1" branch="$2" + [ -n "$branch" ] && git ls-remote --heads "https://github.com/${repo}/elasticsearch-serverless.git" "$branch" 2>/dev/null | grep -q . +} + +if [ -n "${ES_SERVERLESS_BRANCH:-}" ]; then + SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH}" + echo "Using explicit ES_SERVERLESS_BRANCH override: $SERVERLESS_BRANCH" >&2 +else + if [ -n "$PR_AUTHOR_FORK" ] && check_serverless_branch "$PR_AUTHOR_FORK" "$PR_SOURCE"; then + if check_serverless_branch "elastic" "$PR_SOURCE"; then + SERVERLESS_BRANCH="$PR_SOURCE" + echo "Found '$PR_SOURCE' on both $PR_AUTHOR_FORK and elastic; using elastic/" >&2 + else + echo "WARNING: Found '$PR_SOURCE' on $PR_AUTHOR_FORK/elasticsearch-serverless but not on elastic/." >&2 + echo "Push the branch to elastic/ or set ES_SERVERLESS_BRANCH explicitly." >&2 + fi + elif check_serverless_branch "elastic" "$PR_SOURCE"; then + SERVERLESS_BRANCH="$PR_SOURCE" + elif [ "$PR_TARGET" != "main" ] && check_serverless_branch "elastic" "$PR_TARGET"; then + SERVERLESS_BRANCH="$PR_TARGET" + fi +fi +echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 + +# --- Resolve ES submodule commit --- +ES_COMMIT="" +if [ -n "$PR_AUTHOR_FORK" ] && [ -n "$PR_SOURCE" ]; then + ES_COMMIT=$(git ls-remote --heads "https://github.com/${PR_AUTHOR_FORK}/elasticsearch.git" "$PR_SOURCE" 2>/dev/null | awk '{print $1}') + if [ -n "$ES_COMMIT" ]; then + echo "Using ES commit from ${PR_AUTHOR_FORK}/elasticsearch:${PR_SOURCE}" >&2 + fi +fi +if [ -z "$ES_COMMIT" ]; then + ES_COMMIT=$(git ls-remote --heads "https://github.com/elastic/elasticsearch.git" main 2>/dev/null | awk '{print $1}') + ES_COMMIT="${ES_COMMIT:-HEAD}" +fi + +echo "Deploying to serverless QA with custom ml-cpp from PR #${PR_NUM}" >&2 + +cat </dev/null || true)" +PR_SOURCE="$(expr "${BUILDKITE_BRANCH:-}" : '.*:\(.*\)' 2>/dev/null || true)" +PR_TARGET="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" + +# --- Resolve elasticsearch-serverless branch --- +# Reuses the fork/branch resolution pattern from dev-tools/run_es_tests_common.sh. +# The trigger step can only use branches on elastic/elasticsearch-serverless, +# so if a matching branch is found on a fork but not on elastic/, we warn. +SERVERLESS_BRANCH="main" + +check_serverless_branch() { + local repo="$1" branch="$2" + [ -n "$branch" ] && git ls-remote --heads "https://github.com/${repo}/elasticsearch-serverless.git" "$branch" 2>/dev/null | grep -q . +} + +if [ -n "${ES_SERVERLESS_BRANCH:-}" ]; then + SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH}" + echo "Using explicit ES_SERVERLESS_BRANCH override: $SERVERLESS_BRANCH" >&2 +else + if [ -n "$PR_AUTHOR_FORK" ] && check_serverless_branch "$PR_AUTHOR_FORK" "$PR_SOURCE"; then + if check_serverless_branch "elastic" "$PR_SOURCE"; then + SERVERLESS_BRANCH="$PR_SOURCE" + echo "Found '$PR_SOURCE' on both $PR_AUTHOR_FORK and elastic; using elastic/" >&2 + else + echo "WARNING: Found '$PR_SOURCE' on $PR_AUTHOR_FORK/elasticsearch-serverless but not on elastic/." >&2 + echo "The trigger step can only use branches on elastic/elasticsearch-serverless." >&2 + echo "Push the branch to elastic/ or set ES_SERVERLESS_BRANCH explicitly." >&2 + fi + elif check_serverless_branch "elastic" "$PR_SOURCE"; then + SERVERLESS_BRANCH="$PR_SOURCE" + elif [ "$PR_TARGET" != "main" ] && check_serverless_branch "elastic" "$PR_TARGET"; then + SERVERLESS_BRANCH="$PR_TARGET" + fi +fi +echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 + +# --- Resolve ES submodule commit --- +# If the developer has a matching branch on their elasticsearch fork +# (coordinated changes), use that. Otherwise use the latest ES main commit. +ES_COMMIT="" +if [ -n "$PR_AUTHOR_FORK" ] && [ -n "$PR_SOURCE" ]; then + ES_COMMIT=$(git ls-remote --heads "https://github.com/${PR_AUTHOR_FORK}/elasticsearch.git" "$PR_SOURCE" 2>/dev/null | awk '{print $1}') + if [ -n "$ES_COMMIT" ]; then + echo "Using ES commit from ${PR_AUTHOR_FORK}/elasticsearch:${PR_SOURCE}" >&2 + fi +fi +if [ -z "$ES_COMMIT" ]; then + ES_COMMIT=$(git ls-remote --heads "https://github.com/elastic/elasticsearch.git" main 2>/dev/null | awk '{print $1}') + ES_COMMIT="${ES_COMMIT:-HEAD}" +fi + +# --- Resolve ES PR number --- +# The serverless pipeline's PR-specific tests step looks up labels from the +# ES PR. First tries the ml-cpp PR author's matching ES PR (coordinated +# changes), then falls back to any recent open ES PR. +ES_PR_NUM="" +if [ -z "${ELASTICSEARCH_PR_NUMBER:-}" ]; then + if [ -n "$PR_AUTHOR_FORK" ] && [ -n "$PR_SOURCE" ]; then + ES_PR_NUM=$(curl -s "https://api.github.com/repos/elastic/elasticsearch/pulls?head=${PR_AUTHOR_FORK}:${PR_SOURCE}&state=open&per_page=1" 2>/dev/null \ + | python3 -c "import sys,json; prs=json.load(sys.stdin); print(prs[0]['number'] if prs else '')" 2>/dev/null || true) + fi + if [ -z "$ES_PR_NUM" ]; then + ES_PR_NUM=$(curl -s "https://api.github.com/repos/elastic/elasticsearch/pulls?state=open&per_page=1" 2>/dev/null \ + | python3 -c "import sys,json; prs=json.load(sys.stdin); print(prs[0]['number'] if prs else '')" 2>/dev/null || true) + fi +fi +ES_PR_NUM="${ELASTICSEARCH_PR_NUMBER:-${ES_PR_NUM}}" +if [ -z "$ES_PR_NUM" ]; then + echo "WARNING: Could not resolve an ES PR number. The serverless PR-specific tests step may fail." >&2 +fi +echo "Using ES submodule commit: $ES_COMMIT, ES PR number: $ES_PR_NUM" >&2 cat <build|debug|run_qa_tests|run_pytorch_tests|run_serverless_tests)(=(?(?:[^ ]+)))? *(?: for ES_BRANCH=(?([.0-9a-zA-Z]+)))? *(?:with STACK_VERSION=(?([.0-9]+)))? *(?: *on *(?(?:[ ,]*(?:windows|linux|mac(os)?))+))?) *(?(?:[, ]*aarch64|x86_64)+)?$", + "trigger_comment_regex": "^(?:(?:buildkite +)(?build|debug|run_qa_tests|run_pytorch_tests|run_serverless_tests|deploy_serverless_qa)(=(?(?:[^ ]+)))? *(?: for ES_BRANCH=(?([.0-9a-zA-Z]+)))? *(?:with STACK_VERSION=(?([.0-9]+)))? *(?: *on *(?(?:[ ,]*(?:windows|linux|mac(os)?))+))?) *(?(?:[, ]*aarch64|x86_64)+)?$", "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", "skip_ci_labels": ["skip-ci", "jenkins-ci", ">test-mute", ">docs"], "skip_target_branches": ["6.8", "7.11", "7.12"], diff --git a/.buildkite/scripts/steps/build_serverless_docker.sh b/.buildkite/scripts/steps/build_serverless_docker.sh deleted file mode 100755 index 7c53b20509..0000000000 --- a/.buildkite/scripts/steps/build_serverless_docker.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the following additional limitation. Functionality enabled by the -# files subject to the Elastic License 2.0 may only be used in production when -# invoked by an Elasticsearch process with a license key installed that permits -# use of machine learning features. You may not use this file except in -# compliance with the Elastic License 2.0 and the foregoing additional -# limitation. - -# Builds a serverless Elasticsearch Docker image that incorporates custom -# ml-cpp artifacts from the current PR build. -# -# Prerequisites: -# - ml-cpp build artifacts in build/distributions/ml-cpp-*-linux-x86_64.zip -# (downloaded from a prior Buildkite step) -# -# Environment: -# IMAGE_TAG - Docker image tag (required) -# ES_SERVERLESS_BRANCH - elasticsearch-serverless branch to build from (default: main) - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" -ES_SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH:-main}" -DOCKER_REGISTRY="docker.elastic.co/elasticsearch-ci/elasticsearch-serverless" - -VERSION=$(grep '^elasticsearchVersion' "${REPO_ROOT}/gradle.properties" | awk -F= '{ print $2 }' | xargs echo) -VERSION="${VERSION}-SNAPSHOT" - -echo "--- Setting up local Ivy repo with custom ml-cpp ${VERSION}" -IVY_REPO="$(pwd)/ivy-repo" -IVY_ML_DIR="${IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${VERSION}" -mkdir -p "$IVY_ML_DIR" - -cp "build/distributions/ml-cpp-${VERSION}-linux-x86_64.zip" \ - "${IVY_ML_DIR}/ml-cpp-${VERSION}.zip" -cp "build/distributions/ml-cpp-${VERSION}-linux-x86_64.zip" \ - "${IVY_ML_DIR}/ml-cpp-${VERSION}-nodeps.zip" -cp "${REPO_ROOT}/dev-tools/minimal.zip" \ - "${IVY_ML_DIR}/ml-cpp-${VERSION}-deps.zip" - -IVY_REPO_URL="file://${IVY_REPO}" -echo "Ivy repo URL: ${IVY_REPO_URL}" - -echo "--- Obtaining GitHub token for elasticsearch-serverless" -set +x -# ES_SERVERLESS_GITHUB_TOKEN can be provided as a build env var when the -# pipeline's default VAULT_GITHUB_TOKEN lacks access to elasticsearch-serverless. -# Long-term, the ml-cpp pipeline's Vault role should be granted read access to -# a GitHub App token that covers elasticsearch-serverless. -ES_SERVERLESS_TOKEN="${ES_SERVERLESS_GITHUB_TOKEN:-${VAULT_GITHUB_TOKEN:-}}" -if [ -z "${ES_SERVERLESS_TOKEN}" ]; then - echo "ERROR: Could not obtain a GitHub token with access to elasticsearch-serverless." - echo "Set ES_SERVERLESS_GITHUB_TOKEN as a build environment variable." - exit 1 -fi -set -x - -echo "--- Cloning elasticsearch-serverless (branch: ${ES_SERVERLESS_BRANCH})" -cd .. -rm -rf elasticsearch-serverless -set +x -git clone --depth=1 -b "${ES_SERVERLESS_BRANCH}" \ - "https://x-access-token:${ES_SERVERLESS_TOKEN}@github.com/elastic/elasticsearch-serverless.git" -set -x -cd elasticsearch-serverless - -echo "--- Initializing elasticsearch submodule" -set +x -git config url."https://x-access-token:${ES_SERVERLESS_TOKEN}@github.com/".insteadOf "git@github.com:" -set -x -git submodule update --init --depth=1 - -echo "--- Building serverless Docker image with custom ml-cpp" -# The serverless build requires a license key for release builds -LICENSE_KEY=$(mktemp -d)/license.key -vault read -field pubkey \ - secret/ci/elastic-elasticsearch-serverless/migrated/es-license \ - | base64 --decode > "$LICENSE_KEY" - -./gradlew --console=plain --parallel \ - -Dbuild.snapshot=false \ - "-Dlicense.key=${LICENSE_KEY}" \ - "-Dbuild.ml_cpp.repo=${IVY_REPO_URL}" \ - buildDockerImage - -echo "--- Tagging and pushing Docker image" -FULL_TAG="${DOCKER_REGISTRY}:${IMAGE_TAG}" -docker tag elasticsearch-serverless:x86_64 "${FULL_TAG}" - -set +x -DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch-serverless/prod_docker_registry_credentials)" -DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch-serverless/prod_docker_registry_credentials)" -echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin docker.elastic.co -set -x - -docker push "${FULL_TAG}" -echo "Pushed ${FULL_TAG}" - -# Store the image tag as metadata for downstream steps -buildkite-agent meta-data set "serverless-image-tag" "${IMAGE_TAG}" -buildkite-agent meta-data set "serverless-image" "${FULL_TAG}" From f45bc26203182278ade7981ba5fdd991c86933ef Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 20 Apr 2026 13:06:09 +1200 Subject: [PATCH 03/12] Refactor branch/fork selection scripts Reduce duplication by refactoring shell script routines to extract common functionality. --- .../pipelines/deploy_serverless_qa.yml.sh | 26 ++++++----- .../pipelines/run_serverless_tests.yml.sh | 29 ++++++------ dev-tools/run_es_tests_common.sh | 46 +------------------ 3 files changed, 30 insertions(+), 71 deletions(-) diff --git a/.buildkite/pipelines/deploy_serverless_qa.yml.sh b/.buildkite/pipelines/deploy_serverless_qa.yml.sh index 28872f4cbe..eb1c184f9d 100755 --- a/.buildkite/pipelines/deploy_serverless_qa.yml.sh +++ b/.buildkite/pipelines/deploy_serverless_qa.yml.sh @@ -28,7 +28,16 @@ PR_AUTHOR_FORK="$(expr "${BUILDKITE_BRANCH:-}" : '\(.*\):.*' 2>/dev/null || true PR_SOURCE="$(expr "${BUILDKITE_BRANCH:-}" : '.*:\(.*\)' 2>/dev/null || true)" PR_TARGET="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" +ML_CPP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=dev-tools/pick_elasticsearch_clone_target.sh +source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_clone_target.sh" +export PR_AUTHOR="${PR_AUTHOR_FORK}" +export PR_SOURCE_BRANCH="${PR_SOURCE}" +export PR_TARGET_BRANCH="${PR_TARGET}" + # --- Resolve elasticsearch-serverless branch --- +# Same fork/branch *idea* as pick_elasticsearch_clone_target.sh (different repo); +# ES submodule SHA below uses that script directly. SERVERLESS_BRANCH="main" check_serverless_branch() { @@ -56,18 +65,11 @@ else fi echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 -# --- Resolve ES submodule commit --- -ES_COMMIT="" -if [ -n "$PR_AUTHOR_FORK" ] && [ -n "$PR_SOURCE" ]; then - ES_COMMIT=$(git ls-remote --heads "https://github.com/${PR_AUTHOR_FORK}/elasticsearch.git" "$PR_SOURCE" 2>/dev/null | awk '{print $1}') - if [ -n "$ES_COMMIT" ]; then - echo "Using ES commit from ${PR_AUTHOR_FORK}/elasticsearch:${PR_SOURCE}" >&2 - fi -fi -if [ -z "$ES_COMMIT" ]; then - ES_COMMIT=$(git ls-remote --heads "https://github.com/elastic/elasticsearch.git" main 2>/dev/null | awk '{print $1}') - ES_COMMIT="${ES_COMMIT:-HEAD}" -fi +# --- Resolve ES submodule commit (same fork/branch rules as run_es_tests_common.sh) --- +pickCloneTarget || true +ES_COMMIT="$(elasticsearch_selected_branch_head_sha)" +ES_COMMIT="${ES_COMMIT:-HEAD}" +echo "Resolved elasticsearch submodule: ${SELECTED_FORK}/${SELECTED_BRANCH} -> ${ES_COMMIT}" >&2 echo "Deploying to serverless QA with custom ml-cpp from PR #${PR_NUM}" >&2 diff --git a/.buildkite/pipelines/run_serverless_tests.yml.sh b/.buildkite/pipelines/run_serverless_tests.yml.sh index 1323d4f105..48c003e41e 100755 --- a/.buildkite/pipelines/run_serverless_tests.yml.sh +++ b/.buildkite/pipelines/run_serverless_tests.yml.sh @@ -30,8 +30,16 @@ PR_AUTHOR_FORK="$(expr "${BUILDKITE_BRANCH:-}" : '\(.*\):.*' 2>/dev/null || true PR_SOURCE="$(expr "${BUILDKITE_BRANCH:-}" : '.*:\(.*\)' 2>/dev/null || true)" PR_TARGET="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" +ML_CPP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=dev-tools/pick_elasticsearch_clone_target.sh +source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_clone_target.sh" +export PR_AUTHOR="${PR_AUTHOR_FORK}" +export PR_SOURCE_BRANCH="${PR_SOURCE}" +export PR_TARGET_BRANCH="${PR_TARGET}" + # --- Resolve elasticsearch-serverless branch --- -# Reuses the fork/branch resolution pattern from dev-tools/run_es_tests_common.sh. +# Same fork/branch *idea* as pick_elasticsearch_clone_target.sh (different repo); +# ES submodule SHA below uses that script directly. # The trigger step can only use branches on elastic/elasticsearch-serverless, # so if a matching branch is found on a fork but not on elastic/, we warn. SERVERLESS_BRANCH="main" @@ -62,20 +70,11 @@ else fi echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 -# --- Resolve ES submodule commit --- -# If the developer has a matching branch on their elasticsearch fork -# (coordinated changes), use that. Otherwise use the latest ES main commit. -ES_COMMIT="" -if [ -n "$PR_AUTHOR_FORK" ] && [ -n "$PR_SOURCE" ]; then - ES_COMMIT=$(git ls-remote --heads "https://github.com/${PR_AUTHOR_FORK}/elasticsearch.git" "$PR_SOURCE" 2>/dev/null | awk '{print $1}') - if [ -n "$ES_COMMIT" ]; then - echo "Using ES commit from ${PR_AUTHOR_FORK}/elasticsearch:${PR_SOURCE}" >&2 - fi -fi -if [ -z "$ES_COMMIT" ]; then - ES_COMMIT=$(git ls-remote --heads "https://github.com/elastic/elasticsearch.git" main 2>/dev/null | awk '{print $1}') - ES_COMMIT="${ES_COMMIT:-HEAD}" -fi +# --- Resolve ES submodule commit (same fork/branch rules as run_es_tests_common.sh) --- +pickCloneTarget || true +ES_COMMIT="$(elasticsearch_selected_branch_head_sha)" +ES_COMMIT="${ES_COMMIT:-HEAD}" +echo "Resolved elasticsearch submodule: ${SELECTED_FORK}/${SELECTED_BRANCH} -> ${ES_COMMIT}" >&2 # --- Resolve ES PR number --- # The serverless pipeline's PR-specific tests step looks up labels from the diff --git a/dev-tools/run_es_tests_common.sh b/dev-tools/run_es_tests_common.sh index 9dcb5f8509..4419a3c013 100755 --- a/dev-tools/run_es_tests_common.sh +++ b/dev-tools/run_es_tests_common.sh @@ -33,50 +33,8 @@ set -e -function isCloneTargetValid { - FORK_TO_CHECK="$1" - BRANCH_TO_CHECK="$2" - echo "Checking for '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch" - if [ -n "$(git ls-remote --heads "git@github.com:$FORK_TO_CHECK/elasticsearch.git" "$BRANCH_TO_CHECK" 2>/dev/null)" ]; then - echo "Will use '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch for ES integration tests" - return 0 - fi - return 1 -} - -SELECTED_FORK=elastic -SELECTED_BRANCH=main - -function pickCloneTarget { - - if isCloneTargetValid "$GITHUB_PR_OWNER" "$GITHUB_PR_BRANCH" ; then - SELECTED_FORK="$GITHUB_PR_OWNER" - SELECTED_BRANCH="$GITHUB_PR_BRANCH" - return 0 - fi - - if isCloneTargetValid "$PR_AUTHOR" "$PR_SOURCE_BRANCH" ; then - SELECTED_FORK="$PR_AUTHOR" - SELECTED_BRANCH="$PR_SOURCE_BRANCH" - return 0 - fi - - if isCloneTargetValid "$SELECTED_FORK" "$PR_SOURCE_BRANCH" ; then - SELECTED_BRANCH="$PR_SOURCE_BRANCH" - return 0 - fi - - if isCloneTargetValid "$SELECTED_FORK" "$PR_TARGET_BRANCH" ; then - SELECTED_BRANCH="$PR_TARGET_BRANCH" - return 0 - fi - - if isCloneTargetValid "$SELECTED_FORK" "$SELECTED_BRANCH" ; then - return 0 - fi - - return 1 -} +# shellcheck source=pick_elasticsearch_clone_target.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/pick_elasticsearch_clone_target.sh" CLONE_DIR="$1" IVY_REPO_PATH="$2" From c495528de00f02223a884a9e0360026abd1a46ee Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 20 Apr 2026 13:34:27 +1200 Subject: [PATCH 04/12] Pass PROJECT_TYPE env var (default elasticsearch) Allows for setting a different project type than the "elasticsearch" default, e.g. "observability", "security". --- .buildkite/pipelines/deploy_serverless_qa.yml.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipelines/deploy_serverless_qa.yml.sh b/.buildkite/pipelines/deploy_serverless_qa.yml.sh index eb1c184f9d..d6bf83b0cf 100755 --- a/.buildkite/pipelines/deploy_serverless_qa.yml.sh +++ b/.buildkite/pipelines/deploy_serverless_qa.yml.sh @@ -100,4 +100,5 @@ steps: ELASTICSEARCH_SUBMODULE_COMMIT: "${ES_COMMIT}" KEEP_DEPLOYMENT: "${KEEP_DEPLOYMENT:-false}" REGION_ID: "${REGION_ID:-aws-eu-west-1}" + PROJECT_TYPE: "${PROJECT_TYPE:-elasticsearch}" EOL From 32db92284f201f2af293f9d288b7ee1b34bc7755 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 20 Apr 2026 14:20:04 +1200 Subject: [PATCH 05/12] Add missing file --- dev-tools/pick_elasticsearch_clone_target.sh | 78 ++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 dev-tools/pick_elasticsearch_clone_target.sh diff --git a/dev-tools/pick_elasticsearch_clone_target.sh b/dev-tools/pick_elasticsearch_clone_target.sh new file mode 100644 index 0000000000..f1bb8c9dc0 --- /dev/null +++ b/dev-tools/pick_elasticsearch_clone_target.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# + +# Shared logic to choose which elasticsearch fork/branch to use for ml-cpp CI: +# integration test clones (run_es_tests_common.sh) and Buildkite pipelines that +# need ELASTICSEARCH_SUBMODULE_COMMIT without cloning. +# +# Source this file, then call pickCloneTarget. It reads (in order of precedence): +# GITHUB_PR_OWNER / GITHUB_PR_BRANCH — when the job is tied to a GitHub PR +# PR_AUTHOR / PR_SOURCE_BRANCH — fork and branch for coordinated ml-cpp + ES changes +# elastic / PR_SOURCE_BRANCH — upstream branch matching the ml-cpp PR branch name +# elastic / PR_TARGET_BRANCH — target branch of the ml-cpp PR +# elastic / main — final fallback +# +# On success, SELECTED_FORK and SELECTED_BRANCH are set. Optional helper +# elasticsearch_selected_branch_head_sha prints the remote HEAD commit for that +# pair (same transport as isCloneTargetValid: git@github.com). +# +# This file must be sourced (not executed) so that SELECTED_* remain in the caller's shell. + +function isCloneTargetValid { + FORK_TO_CHECK="$1" + BRANCH_TO_CHECK="$2" + echo "Checking for '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch" + if [ -n "$(git ls-remote --heads "git@github.com:$FORK_TO_CHECK/elasticsearch.git" "$BRANCH_TO_CHECK" 2>/dev/null)" ]; then + echo "Will use '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch for ES integration tests" + return 0 + fi + return 1 +} + +SELECTED_FORK=elastic +SELECTED_BRANCH=main + +function pickCloneTarget { + + if isCloneTargetValid "$GITHUB_PR_OWNER" "$GITHUB_PR_BRANCH" ; then + SELECTED_FORK="$GITHUB_PR_OWNER" + SELECTED_BRANCH="$GITHUB_PR_BRANCH" + return 0 + fi + + if isCloneTargetValid "$PR_AUTHOR" "$PR_SOURCE_BRANCH" ; then + SELECTED_FORK="$PR_AUTHOR" + SELECTED_BRANCH="$PR_SOURCE_BRANCH" + return 0 + fi + + if isCloneTargetValid "$SELECTED_FORK" "$PR_SOURCE_BRANCH" ; then + SELECTED_BRANCH="$PR_SOURCE_BRANCH" + return 0 + fi + + if isCloneTargetValid "$SELECTED_FORK" "$PR_TARGET_BRANCH" ; then + SELECTED_BRANCH="$PR_TARGET_BRANCH" + return 0 + fi + + if isCloneTargetValid "$SELECTED_FORK" "$SELECTED_BRANCH" ; then + return 0 + fi + + return 1 +} + +# Prints the commit SHA at the head of SELECTED_BRANCH on SELECTED_FORK, or empty if unavailable. +function elasticsearch_selected_branch_head_sha { + git ls-remote --heads "git@github.com:${SELECTED_FORK}/elasticsearch.git" "${SELECTED_BRANCH}" 2>/dev/null | awk '{print $1; exit}' +} From 500f01158ba30a803c463ece43a019d4cbfa6d7b Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Tue, 21 Apr 2026 14:51:38 +1200 Subject: [PATCH 06/12] Fix bug affecting yaml parsing --- dev-tools/pick_elasticsearch_clone_target.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dev-tools/pick_elasticsearch_clone_target.sh b/dev-tools/pick_elasticsearch_clone_target.sh index f1bb8c9dc0..900e6f019c 100644 --- a/dev-tools/pick_elasticsearch_clone_target.sh +++ b/dev-tools/pick_elasticsearch_clone_target.sh @@ -30,9 +30,11 @@ function isCloneTargetValid { FORK_TO_CHECK="$1" BRANCH_TO_CHECK="$2" - echo "Checking for '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch" + # Diagnostics must go to stderr: callers (e.g. deploy_serverless_qa.yml.sh) + # pipe stdout to `buildkite-agent pipeline upload` and expect only YAML. + echo "Checking for '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch" >&2 if [ -n "$(git ls-remote --heads "git@github.com:$FORK_TO_CHECK/elasticsearch.git" "$BRANCH_TO_CHECK" 2>/dev/null)" ]; then - echo "Will use '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch for ES integration tests" + echo "Will use '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch for ES integration tests" >&2 return 0 fi return 1 From 8c5078728b8b4208739086cbb406f57c139a661f Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 23 Apr 2026 11:10:30 +1200 Subject: [PATCH 07/12] Pass ML_CPP_COMMIT for serverless Docker tag discrimination Expose the ml-cpp repo commit to the triggered serverless build so pipeline.pr.yml.sh can fold it into IMAGE_TAG_SUFFIX with ML_CPP_BUILD_ID. Made-with: Cursor --- .buildkite/pipelines/run_serverless_tests.yml.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/pipelines/run_serverless_tests.yml.sh b/.buildkite/pipelines/run_serverless_tests.yml.sh index 48c003e41e..bd0279e927 100755 --- a/.buildkite/pipelines/run_serverless_tests.yml.sh +++ b/.buildkite/pipelines/run_serverless_tests.yml.sh @@ -122,6 +122,9 @@ steps: env: UPDATE_SUBMODULE: "false" ML_CPP_BUILD_ID: "${BUILDKITE_BUILD_ID}" + # ml-cpp repo commit at trigger time; serverless folds this into IMAGE_TAG + # with ML_CPP_BUILD_ID so Docker tags never collide with stock builds. + ML_CPP_COMMIT: "${BUILDKITE_COMMIT}" ELASTICSEARCH_SUBMODULE_COMMIT: "${ES_COMMIT}" ELASTICSEARCH_PR_NUMBER: "${ES_PR_NUM}" EOL From 651ff54823e78ec3293be63bfeef4c7ae097483e Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 24 Apr 2026 15:40:56 +1200 Subject: [PATCH 08/12] [ML] Pass serverless options from PR comment tail Extend trigger_comment_regex with optional serverless_kv capture (space- separated KEY=value tokens). Parse GITHUB_PR_COMMENT_VAR_SERVERLESS_KV in config and whitelist KEEP_DEPLOYMENT, REGION_ID, PROJECT_TYPE, ES_SERVERLESS_BRANCH into os.environ. Forward those to pipeline env when serverless deploy or test steps are selected. Made-with: Cursor --- .buildkite/ml_pipeline/config.py | 30 ++++++++++++++++++++++++++++++ .buildkite/pipeline.json.py | 11 +++++++++++ .buildkite/pull-requests.json | 2 +- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.buildkite/ml_pipeline/config.py b/.buildkite/ml_pipeline/config.py index 184ecc93d1..f3ff565872 100644 --- a/.buildkite/ml_pipeline/config.py +++ b/.buildkite/ml_pipeline/config.py @@ -11,6 +11,17 @@ import os import re +# Keys allowed in the optional tail of trigger_comment_regex (group serverless_kv). +_SERVERLESS_KV_KEYS = frozenset( + { + "KEEP_DEPLOYMENT", + "REGION_ID", + "PROJECT_TYPE", + "ES_SERVERLESS_BRANCH", + } +) + + class Config: build_windows: bool = False build_macos: bool = False @@ -44,6 +55,8 @@ def parse_comment(self): if self.run_pytorch_tests or self.run_qa_tests or self.run_serverless_tests or self.deploy_serverless_qa: self.action = "build" + self._apply_serverless_kv_from_comment() + # If the ACTION is set to "run_qa_tests" then set some optional variables governing the ES branch to build, the # stack version to set and the subset of QA tests to run, depending on whether appropriate variables are set in # the environment. @@ -177,3 +190,20 @@ def parse(self): self.build_x86_64 = "--build-x86_64" self.run_qa_tests = False + def _apply_serverless_kv_from_comment(self): + """Copy whitelisted KEY=value tokens from the PR comment regex capture into os.environ.""" + + env_key = "GITHUB_PR_COMMENT_VAR_SERVERLESS_KV" + if env_key not in os.environ: + return + raw = os.environ[env_key].strip() + if not raw: + return + for token in raw.split(): + key, sep, value = token.partition("=") + if not sep or key not in _SERVERLESS_KV_KEYS: + continue + if key == "KEEP_DEPLOYMENT" and value.lower() not in ("true", "false"): + continue + os.environ[key] = value + diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 0a0a9406cb..e524d941dd 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -17,6 +17,7 @@ # import json +import os from ml_pipeline import ( step, @@ -52,6 +53,16 @@ def main(): "VERSION_QUALIFIER": "", "ML_BUILD_STEP_KEYS": ",".join(build_step_keys), } + if config.run_serverless_tests or config.deploy_serverless_qa: + for serverless_env_key in ( + "KEEP_DEPLOYMENT", + "REGION_ID", + "PROJECT_TYPE", + "ES_SERVERLESS_BRANCH", + ): + value = os.environ.get(serverless_env_key) + if value: + env[serverless_env_key] = value if config.build_windows: build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 2cb5ad394b..ec1fcba9c4 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -9,7 +9,7 @@ "commit_status_context": "ml-cpp-ci", "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^(?:(?:buildkite +)(?build|debug|run_qa_tests|run_pytorch_tests|run_serverless_tests|deploy_serverless_qa)(=(?(?:[^ ]+)))? *(?: for ES_BRANCH=(?([.0-9a-zA-Z]+)))? *(?:with STACK_VERSION=(?([.0-9]+)))? *(?: *on *(?(?:[ ,]*(?:windows|linux|mac(os)?))+))?) *(?(?:[, ]*aarch64|x86_64)+)?$", + "trigger_comment_regex": "^(?:(?:buildkite +)(?build|debug|run_qa_tests|run_pytorch_tests|run_serverless_tests|deploy_serverless_qa)(=(?(?:[^ ]+)))? *(?: for ES_BRANCH=(?([.0-9a-zA-Z]+)))? *(?:with STACK_VERSION=(?([.0-9]+)))? *(?: *on *(?(?:[ ,]*(?:windows|linux|mac(os)?))+))?) *(?(?:[, ]*aarch64|x86_64)+)?(?: *(?(?:(?:KEEP_DEPLOYMENT|REGION_ID|PROJECT_TYPE|ES_SERVERLESS_BRANCH)=[^\\s]+)(?: +(?:KEEP_DEPLOYMENT|REGION_ID|PROJECT_TYPE|ES_SERVERLESS_BRANCH)=[^\\s]+)*))?$", "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", "skip_ci_labels": ["skip-ci", "jenkins-ci", ">test-mute", ">docs"], "skip_target_branches": ["6.8", "7.11", "7.12"], From f8e3f2c5c2b6c1a3db935360e7657141cd842eaf Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Tue, 28 Apr 2026 10:02:50 +1200 Subject: [PATCH 09/12] [ML] Factor serverless branch selection into dev-tools helper Extract duplicate elasticsearch-serverless branch resolution from run_serverless_tests.yml.sh and deploy_serverless_qa.yml.sh into pick_elasticsearch_serverless_branch.sh, alongside pick_elasticsearch_clone_target.sh for the elasticsearch repo. Addresses PR 3027 review feedback on duplicated fork/branch logic. Made-with: Cursor --- .../pipelines/deploy_serverless_qa.yml.sh | 35 ++--------- .../pipelines/run_serverless_tests.yml.sh | 38 ++---------- .../pick_elasticsearch_serverless_branch.sh | 61 +++++++++++++++++++ 3 files changed, 71 insertions(+), 63 deletions(-) create mode 100644 dev-tools/pick_elasticsearch_serverless_branch.sh diff --git a/.buildkite/pipelines/deploy_serverless_qa.yml.sh b/.buildkite/pipelines/deploy_serverless_qa.yml.sh index d6bf83b0cf..3390f740d9 100755 --- a/.buildkite/pipelines/deploy_serverless_qa.yml.sh +++ b/.buildkite/pipelines/deploy_serverless_qa.yml.sh @@ -35,37 +35,12 @@ export PR_AUTHOR="${PR_AUTHOR_FORK}" export PR_SOURCE_BRANCH="${PR_SOURCE}" export PR_TARGET_BRANCH="${PR_TARGET}" -# --- Resolve elasticsearch-serverless branch --- -# Same fork/branch *idea* as pick_elasticsearch_clone_target.sh (different repo); -# ES submodule SHA below uses that script directly. -SERVERLESS_BRANCH="main" +# --- Resolve elasticsearch-serverless branch (shared with run_serverless_tests.yml.sh) --- +# shellcheck source=dev-tools/pick_elasticsearch_serverless_branch.sh +source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_serverless_branch.sh" +pickElasticsearchServerlessBranch -check_serverless_branch() { - local repo="$1" branch="$2" - [ -n "$branch" ] && git ls-remote --heads "https://github.com/${repo}/elasticsearch-serverless.git" "$branch" 2>/dev/null | grep -q . -} - -if [ -n "${ES_SERVERLESS_BRANCH:-}" ]; then - SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH}" - echo "Using explicit ES_SERVERLESS_BRANCH override: $SERVERLESS_BRANCH" >&2 -else - if [ -n "$PR_AUTHOR_FORK" ] && check_serverless_branch "$PR_AUTHOR_FORK" "$PR_SOURCE"; then - if check_serverless_branch "elastic" "$PR_SOURCE"; then - SERVERLESS_BRANCH="$PR_SOURCE" - echo "Found '$PR_SOURCE' on both $PR_AUTHOR_FORK and elastic; using elastic/" >&2 - else - echo "WARNING: Found '$PR_SOURCE' on $PR_AUTHOR_FORK/elasticsearch-serverless but not on elastic/." >&2 - echo "Push the branch to elastic/ or set ES_SERVERLESS_BRANCH explicitly." >&2 - fi - elif check_serverless_branch "elastic" "$PR_SOURCE"; then - SERVERLESS_BRANCH="$PR_SOURCE" - elif [ "$PR_TARGET" != "main" ] && check_serverless_branch "elastic" "$PR_TARGET"; then - SERVERLESS_BRANCH="$PR_TARGET" - fi -fi -echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 - -# --- Resolve ES submodule commit (same fork/branch rules as run_es_tests_common.sh) --- +# --- Resolve ES submodule commit (shared pick_elasticsearch_clone_target.sh) --- pickCloneTarget || true ES_COMMIT="$(elasticsearch_selected_branch_head_sha)" ES_COMMIT="${ES_COMMIT:-HEAD}" diff --git a/.buildkite/pipelines/run_serverless_tests.yml.sh b/.buildkite/pipelines/run_serverless_tests.yml.sh index bd0279e927..91eca65af0 100755 --- a/.buildkite/pipelines/run_serverless_tests.yml.sh +++ b/.buildkite/pipelines/run_serverless_tests.yml.sh @@ -37,40 +37,12 @@ export PR_AUTHOR="${PR_AUTHOR_FORK}" export PR_SOURCE_BRANCH="${PR_SOURCE}" export PR_TARGET_BRANCH="${PR_TARGET}" -# --- Resolve elasticsearch-serverless branch --- -# Same fork/branch *idea* as pick_elasticsearch_clone_target.sh (different repo); -# ES submodule SHA below uses that script directly. -# The trigger step can only use branches on elastic/elasticsearch-serverless, -# so if a matching branch is found on a fork but not on elastic/, we warn. -SERVERLESS_BRANCH="main" +# --- Resolve elasticsearch-serverless branch (shared with deploy_serverless_qa.yml.sh) --- +# shellcheck source=dev-tools/pick_elasticsearch_serverless_branch.sh +source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_serverless_branch.sh" +pickElasticsearchServerlessBranch -check_serverless_branch() { - local repo="$1" branch="$2" - [ -n "$branch" ] && git ls-remote --heads "https://github.com/${repo}/elasticsearch-serverless.git" "$branch" 2>/dev/null | grep -q . -} - -if [ -n "${ES_SERVERLESS_BRANCH:-}" ]; then - SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH}" - echo "Using explicit ES_SERVERLESS_BRANCH override: $SERVERLESS_BRANCH" >&2 -else - if [ -n "$PR_AUTHOR_FORK" ] && check_serverless_branch "$PR_AUTHOR_FORK" "$PR_SOURCE"; then - if check_serverless_branch "elastic" "$PR_SOURCE"; then - SERVERLESS_BRANCH="$PR_SOURCE" - echo "Found '$PR_SOURCE' on both $PR_AUTHOR_FORK and elastic; using elastic/" >&2 - else - echo "WARNING: Found '$PR_SOURCE' on $PR_AUTHOR_FORK/elasticsearch-serverless but not on elastic/." >&2 - echo "The trigger step can only use branches on elastic/elasticsearch-serverless." >&2 - echo "Push the branch to elastic/ or set ES_SERVERLESS_BRANCH explicitly." >&2 - fi - elif check_serverless_branch "elastic" "$PR_SOURCE"; then - SERVERLESS_BRANCH="$PR_SOURCE" - elif [ "$PR_TARGET" != "main" ] && check_serverless_branch "elastic" "$PR_TARGET"; then - SERVERLESS_BRANCH="$PR_TARGET" - fi -fi -echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 - -# --- Resolve ES submodule commit (same fork/branch rules as run_es_tests_common.sh) --- +# --- Resolve ES submodule commit (shared pick_elasticsearch_clone_target.sh) --- pickCloneTarget || true ES_COMMIT="$(elasticsearch_selected_branch_head_sha)" ES_COMMIT="${ES_COMMIT:-HEAD}" diff --git a/dev-tools/pick_elasticsearch_serverless_branch.sh b/dev-tools/pick_elasticsearch_serverless_branch.sh new file mode 100644 index 0000000000..e546d511bd --- /dev/null +++ b/dev-tools/pick_elasticsearch_serverless_branch.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# + +# Choose which branch of elastic/elasticsearch-serverless to pass to Buildkite +# trigger steps (es-pr-check, deploy-qa). This mirrors the fork/branch *idea* +# behind pick_elasticsearch_clone_target.sh, but for the serverless repo. +# +# Source this file after setting (from ml-cpp PR metadata): +# PR_AUTHOR_FORK — fork owner from BUILDKITE_BRANCH (author before ':') +# PR_SOURCE — branch name from BUILDKITE_BRANCH (after ':') +# PR_TARGET — BUILDKITE_PULL_REQUEST_BASE_BRANCH (default main) +# Optional override: +# ES_SERVERLESS_BRANCH — force this branch name +# +# Call pickElasticsearchServerlessBranch. It sets SERVERLESS_BRANCH and writes +# diagnostics to stderr (callers often pipe stdout to buildkite-agent). + +SERVERLESS_BRANCH="main" + +function isElasticsearchServerlessBranchAtRemote { + local repo="$1" + local branch="$2" + [ -n "$branch" ] && git ls-remote --heads "https://github.com/${repo}/elasticsearch-serverless.git" "$branch" 2>/dev/null | grep -q . +} + +function pickElasticsearchServerlessBranch { + SERVERLESS_BRANCH="main" + + if [ -n "${ES_SERVERLESS_BRANCH:-}" ]; then + SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH}" + echo "Using explicit ES_SERVERLESS_BRANCH override: $SERVERLESS_BRANCH" >&2 + echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 + return 0 + fi + + if [ -n "$PR_AUTHOR_FORK" ] && isElasticsearchServerlessBranchAtRemote "$PR_AUTHOR_FORK" "$PR_SOURCE"; then + if isElasticsearchServerlessBranchAtRemote "elastic" "$PR_SOURCE"; then + SERVERLESS_BRANCH="$PR_SOURCE" + echo "Found '$PR_SOURCE' on both $PR_AUTHOR_FORK and elastic; using elastic/" >&2 + else + echo "WARNING: Found '$PR_SOURCE' on $PR_AUTHOR_FORK/elasticsearch-serverless but not on elastic/." >&2 + echo "The trigger step can only use branches on elastic/elasticsearch-serverless." >&2 + echo "Push the branch to elastic/ or set ES_SERVERLESS_BRANCH explicitly." >&2 + fi + elif isElasticsearchServerlessBranchAtRemote "elastic" "$PR_SOURCE"; then + SERVERLESS_BRANCH="$PR_SOURCE" + elif [ "$PR_TARGET" != "main" ] && isElasticsearchServerlessBranchAtRemote "elastic" "$PR_TARGET"; then + SERVERLESS_BRANCH="$PR_TARGET" + fi + + echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 +} From e39fd4074d3f7d4bcb3af63f4b3cda1f543e2f2d Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 30 Apr 2026 16:18:21 +1200 Subject: [PATCH 10/12] [ML] Address Copilot review on serverless CI helpers - Validate ES_SERVERLESS_BRANCH exists on elastic before accepting override - isCloneTargetValid: reject empty fork/branch; use local vars when sourced - Force Linux + both CPU arches for serverless actions after parsing PR comment - Gate serverless pipeline uploads on both Linux arch build flags - Forward KEEP_DEPLOYMENT/REGION_ID/PROJECT_TYPE to serverless test trigger with YAML-safe escaping - Escape deploy QA trigger env values for generated YAML; fail fast on bad serverless branch Made-with: Cursor --- .buildkite/ml_pipeline/config.py | 12 ++++++++++++ .buildkite/pipeline.json.py | 9 ++++++--- .buildkite/pipelines/deploy_serverless_qa.yml.sh | 15 +++++++++++---- .buildkite/pipelines/run_serverless_tests.yml.sh | 12 +++++++++++- dev-tools/pick_elasticsearch_clone_target.sh | 13 ++++++++----- dev-tools/pick_elasticsearch_serverless_branch.sh | 13 +++++++++---- 6 files changed, 57 insertions(+), 17 deletions(-) diff --git a/.buildkite/ml_pipeline/config.py b/.buildkite/ml_pipeline/config.py index f3ff565872..8b13ec39d3 100644 --- a/.buildkite/ml_pipeline/config.py +++ b/.buildkite/ml_pipeline/config.py @@ -110,6 +110,14 @@ def parse_comment(self): self.build_macos = True self.build_linux = True + # Serverless runner pipelines depend on both Linux aarch64 and x86_64 + # build steps. Normalize after platform/arch parsing so PR comment tails + # cannot leave dangling depends_on keys or skip Linux builds. + if self.run_serverless_tests or self.deploy_serverless_qa: + self.build_aarch64 = "--build-aarch64" + self.build_x86_64 = "--build-x86_64" + self.build_linux = True + # If no explicit action was set (e.g. "buildkite test this" via # always_trigger_comment_regex), check PR labels for QA/PyTorch # flags. This is done after platform/arch defaults so that @@ -205,5 +213,9 @@ def _apply_serverless_kv_from_comment(self): continue if key == "KEEP_DEPLOYMENT" and value.lower() not in ("true", "false"): continue + if key in ("REGION_ID", "PROJECT_TYPE") and not re.fullmatch(r"[A-Za-z0-9_.:-]+", value): + continue + if key == "ES_SERVERLESS_BRANCH" and not re.fullmatch(r"[A-Za-z0-9_./-]+", value): + continue os.environ[key] = value diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index e524d941dd..eff889d8a5 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -90,11 +90,14 @@ def main(): pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline", ".buildkite/pipelines/run_es_tests_aarch64.yml.sh")) - # Serverless tests require both x86_64 and aarch64 Linux builds. - if config.run_serverless_tests: + # Serverless tests/deploy require both Linux aarch64 and x86_64 build steps. + linux_both_arches = ( + config.build_linux and config.build_aarch64 and config.build_x86_64 + ) + if linux_both_arches and config.run_serverless_tests: pipeline_steps.append(pipeline_steps.generate_step("Upload serverless tests runner pipeline", ".buildkite/pipelines/run_serverless_tests.yml.sh")) - if config.deploy_serverless_qa: + if linux_both_arches and config.deploy_serverless_qa: pipeline_steps.append(pipeline_steps.generate_step("Upload serverless QA deploy pipeline", ".buildkite/pipelines/deploy_serverless_qa.yml.sh")) diff --git a/.buildkite/pipelines/deploy_serverless_qa.yml.sh b/.buildkite/pipelines/deploy_serverless_qa.yml.sh index 3390f740d9..4b89ef2e63 100755 --- a/.buildkite/pipelines/deploy_serverless_qa.yml.sh +++ b/.buildkite/pipelines/deploy_serverless_qa.yml.sh @@ -38,7 +38,7 @@ export PR_TARGET_BRANCH="${PR_TARGET}" # --- Resolve elasticsearch-serverless branch (shared with run_serverless_tests.yml.sh) --- # shellcheck source=dev-tools/pick_elasticsearch_serverless_branch.sh source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_serverless_branch.sh" -pickElasticsearchServerlessBranch +pickElasticsearchServerlessBranch || exit 1 # --- Resolve ES submodule commit (shared pick_elasticsearch_clone_target.sh) --- pickCloneTarget || true @@ -48,6 +48,13 @@ echo "Resolved elasticsearch submodule: ${SELECTED_FORK}/${SELECTED_BRANCH} -> $ echo "Deploying to serverless QA with custom ml-cpp from PR #${PR_NUM}" >&2 +yaml_double_quote_escape() { + printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' +} +KEEP_DEPLOYMENT_SAFE=$(yaml_double_quote_escape "${KEEP_DEPLOYMENT:-false}") +REGION_ID_SAFE=$(yaml_double_quote_escape "${REGION_ID:-aws-eu-west-1}") +PROJECT_TYPE_SAFE=$(yaml_double_quote_escape "${PROJECT_TYPE:-elasticsearch}") + cat <&2 +yaml_double_quote_escape() { + printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' +} +KEEP_DEPLOYMENT_SAFE=$(yaml_double_quote_escape "${KEEP_DEPLOYMENT:-false}") +REGION_ID_SAFE=$(yaml_double_quote_escape "${REGION_ID:-aws-eu-west-1}") +PROJECT_TYPE_SAFE=$(yaml_double_quote_escape "${PROJECT_TYPE:-elasticsearch}") + cat <&2 - if [ -n "$(git ls-remote --heads "git@github.com:$FORK_TO_CHECK/elasticsearch.git" "$BRANCH_TO_CHECK" 2>/dev/null)" ]; then - echo "Will use '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch for ES integration tests" >&2 + echo "Checking for '$branch_to_check' branch at $fork_to_check/elasticsearch" >&2 + if [ -n "$(git ls-remote --heads "git@github.com:${fork_to_check}/elasticsearch.git" "$branch_to_check" 2>/dev/null)" ]; then + echo "Will use '$branch_to_check' branch at $fork_to_check/elasticsearch for ES integration tests" >&2 return 0 fi return 1 diff --git a/dev-tools/pick_elasticsearch_serverless_branch.sh b/dev-tools/pick_elasticsearch_serverless_branch.sh index e546d511bd..e26492f411 100644 --- a/dev-tools/pick_elasticsearch_serverless_branch.sh +++ b/dev-tools/pick_elasticsearch_serverless_branch.sh @@ -36,10 +36,15 @@ function pickElasticsearchServerlessBranch { SERVERLESS_BRANCH="main" if [ -n "${ES_SERVERLESS_BRANCH:-}" ]; then - SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH}" - echo "Using explicit ES_SERVERLESS_BRANCH override: $SERVERLESS_BRANCH" >&2 - echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 - return 0 + if isElasticsearchServerlessBranchAtRemote "elastic" "${ES_SERVERLESS_BRANCH}"; then + SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH}" + echo "Using explicit ES_SERVERLESS_BRANCH override: $SERVERLESS_BRANCH" >&2 + echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 + return 0 + fi + echo "ERROR: ES_SERVERLESS_BRANCH override '${ES_SERVERLESS_BRANCH}' was not found on elastic/elasticsearch-serverless." >&2 + echo "Set ES_SERVERLESS_BRANCH to an existing branch on elastic/elasticsearch-serverless." >&2 + return 1 fi if [ -n "$PR_AUTHOR_FORK" ] && isElasticsearchServerlessBranchAtRemote "$PR_AUTHOR_FORK" "$PR_SOURCE"; then From b907a1fbc790d5aafafd87610ddac3d0f1a73372 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 30 Apr 2026 16:27:20 +1200 Subject: [PATCH 11/12] [ML] Factor serverless Buildkite trigger prep into dev-tools helper Move shared SAFE_MESSAGE/PR metadata, pick_* wiring, ES commit resolution, YAML-safe QA env vars, and the upload_ml_cpp_deps step fragment into dev-tools/serverless_buildkite_trigger_prepare.sh so deploy and es-pr-check pipelines stay in sync. Made-with: Cursor --- .../pipelines/deploy_serverless_qa.yml.sh | 51 ++---------- .../pipelines/run_serverless_tests.yml.sh | 50 ++---------- .../serverless_buildkite_trigger_prepare.sh | 80 +++++++++++++++++++ 3 files changed, 90 insertions(+), 91 deletions(-) create mode 100644 dev-tools/serverless_buildkite_trigger_prepare.sh diff --git a/.buildkite/pipelines/deploy_serverless_qa.yml.sh b/.buildkite/pipelines/deploy_serverless_qa.yml.sh index 4b89ef2e63..c3d0eea318 100755 --- a/.buildkite/pipelines/deploy_serverless_qa.yml.sh +++ b/.buildkite/pipelines/deploy_serverless_qa.yml.sh @@ -17,59 +17,18 @@ # (via the Buildkite UI) to keep it longer. The build annotations will # contain the URL and encrypted credentials for accessing the deployment. -SAFE_MESSAGE=$(printf '%s' "${BUILDKITE_MESSAGE}" | head -1 | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g') -PR_NUM="${BUILDKITE_PULL_REQUEST}" -if [ -z "${PR_NUM}" ] || [ "${PR_NUM}" = "false" ]; then - PR_NUM="manual" -fi - -# Extract PR metadata once for reuse. -PR_AUTHOR_FORK="$(expr "${BUILDKITE_BRANCH:-}" : '\(.*\):.*' 2>/dev/null || true)" -PR_SOURCE="$(expr "${BUILDKITE_BRANCH:-}" : '.*:\(.*\)' 2>/dev/null || true)" -PR_TARGET="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" - ML_CPP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -# shellcheck source=dev-tools/pick_elasticsearch_clone_target.sh -source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_clone_target.sh" -export PR_AUTHOR="${PR_AUTHOR_FORK}" -export PR_SOURCE_BRANCH="${PR_SOURCE}" -export PR_TARGET_BRANCH="${PR_TARGET}" - -# --- Resolve elasticsearch-serverless branch (shared with run_serverless_tests.yml.sh) --- -# shellcheck source=dev-tools/pick_elasticsearch_serverless_branch.sh -source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_serverless_branch.sh" -pickElasticsearchServerlessBranch || exit 1 +# shellcheck source=dev-tools/serverless_buildkite_trigger_prepare.sh +source "${ML_CPP_ROOT}/dev-tools/serverless_buildkite_trigger_prepare.sh" -# --- Resolve ES submodule commit (shared pick_elasticsearch_clone_target.sh) --- -pickCloneTarget || true -ES_COMMIT="$(elasticsearch_selected_branch_head_sha)" -ES_COMMIT="${ES_COMMIT:-HEAD}" -echo "Resolved elasticsearch submodule: ${SELECTED_FORK}/${SELECTED_BRANCH} -> ${ES_COMMIT}" >&2 +prepareMlCppServerlessTriggerContext "${BASH_SOURCE[0]}" || exit 1 +assignServerlessQaTriggerEnvYamlEscapes echo "Deploying to serverless QA with custom ml-cpp from PR #${PR_NUM}" >&2 -yaml_double_quote_escape() { - printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -} -KEEP_DEPLOYMENT_SAFE=$(yaml_double_quote_escape "${KEEP_DEPLOYMENT:-false}") -REGION_ID_SAFE=$(yaml_double_quote_escape "${REGION_ID:-aws-eu-west-1}") -PROJECT_TYPE_SAFE=$(yaml_double_quote_escape "${PROJECT_TYPE:-elasticsearch}") - cat </dev/null || true)" -PR_SOURCE="$(expr "${BUILDKITE_BRANCH:-}" : '.*:\(.*\)' 2>/dev/null || true)" -PR_TARGET="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" - ML_CPP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -# shellcheck source=dev-tools/pick_elasticsearch_clone_target.sh -source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_clone_target.sh" -export PR_AUTHOR="${PR_AUTHOR_FORK}" -export PR_SOURCE_BRANCH="${PR_SOURCE}" -export PR_TARGET_BRANCH="${PR_TARGET}" +# shellcheck source=dev-tools/serverless_buildkite_trigger_prepare.sh +source "${ML_CPP_ROOT}/dev-tools/serverless_buildkite_trigger_prepare.sh" -# --- Resolve elasticsearch-serverless branch (shared with deploy_serverless_qa.yml.sh) --- -# shellcheck source=dev-tools/pick_elasticsearch_serverless_branch.sh -source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_serverless_branch.sh" -pickElasticsearchServerlessBranch || exit 1 - -# --- Resolve ES submodule commit (shared pick_elasticsearch_clone_target.sh) --- -pickCloneTarget || true -ES_COMMIT="$(elasticsearch_selected_branch_head_sha)" -ES_COMMIT="${ES_COMMIT:-HEAD}" -echo "Resolved elasticsearch submodule: ${SELECTED_FORK}/${SELECTED_BRANCH} -> ${ES_COMMIT}" >&2 +prepareMlCppServerlessTriggerContext "${BASH_SOURCE[0]}" || exit 1 # --- Resolve ES PR number --- # The serverless pipeline's PR-specific tests step looks up labels from the @@ -69,28 +46,11 @@ if [ -z "$ES_PR_NUM" ]; then fi echo "Using ES submodule commit: $ES_COMMIT, ES PR number: $ES_PR_NUM" >&2 -yaml_double_quote_escape() { - printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -} -KEEP_DEPLOYMENT_SAFE=$(yaml_double_quote_escape "${KEEP_DEPLOYMENT:-false}") -REGION_ID_SAFE=$(yaml_double_quote_escape "${REGION_ID:-aws-eu-west-1}") -PROJECT_TYPE_SAFE=$(yaml_double_quote_escape "${PROJECT_TYPE:-elasticsearch}") +assignServerlessQaTriggerEnvYamlEscapes cat <&2 + return 1 + fi + + SAFE_MESSAGE=$(printf '%s' "${BUILDKITE_MESSAGE}" | head -1 | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g') + PR_NUM="${BUILDKITE_PULL_REQUEST}" + if [ -z "${PR_NUM}" ] || [ "${PR_NUM}" = "false" ]; then + PR_NUM="manual" + fi + + PR_AUTHOR_FORK="$(expr "${BUILDKITE_BRANCH:-}" : '\(.*\):.*' 2>/dev/null || true)" + PR_SOURCE="$(expr "${BUILDKITE_BRANCH:-}" : '.*:\(.*\)' 2>/dev/null || true)" + PR_TARGET="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" + + ML_CPP_ROOT="$(cd "$(dirname "${pipeline_script}")/../.." && pwd)" + # shellcheck source=dev-tools/pick_elasticsearch_clone_target.sh + source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_clone_target.sh" + export PR_AUTHOR="${PR_AUTHOR_FORK}" + export PR_SOURCE_BRANCH="${PR_SOURCE}" + export PR_TARGET_BRANCH="${PR_TARGET}" + + # shellcheck source=dev-tools/pick_elasticsearch_serverless_branch.sh + source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_serverless_branch.sh" + pickElasticsearchServerlessBranch || return 1 + + pickCloneTarget || true + ES_COMMIT="$(elasticsearch_selected_branch_head_sha)" + ES_COMMIT="${ES_COMMIT:-HEAD}" + echo "Resolved elasticsearch submodule: ${SELECTED_FORK}/${SELECTED_BRANCH} -> ${ES_COMMIT}" >&2 +} + +function emitServerlessUploadMlCppDepsStepYaml { + cat <<'EOS' + - label: ":package: Upload ml-cpp deps artifact" + key: "upload_ml_cpp_deps" + command: 'buildkite-agent artifact upload dev-tools/minimal.zip' + depends_on: + - "build_test_linux-x86_64-RelWithDebInfo" + - "build_test_linux-aarch64-RelWithDebInfo" + agents: + provider: aws + instanceType: m6i.xlarge + imagePrefix: core-amazonlinux-2023 + diskSizeGb: 100 + diskName: '/dev/xvda' + +EOS +} + From 067312553e785039cf6ed2679e36b16ac1ff6180 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 1 May 2026 11:13:45 +1200 Subject: [PATCH 12/12] Sequential serverless branch resolution Resolve elasticsearch-serverless branch in ordered steps so a fork-only PR_SOURCE warning does not skip the elastic PR_TARGET fallback. Made-with: Cursor --- .../pick_elasticsearch_serverless_branch.sh | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/dev-tools/pick_elasticsearch_serverless_branch.sh b/dev-tools/pick_elasticsearch_serverless_branch.sh index e26492f411..5b3dcff418 100644 --- a/dev-tools/pick_elasticsearch_serverless_branch.sh +++ b/dev-tools/pick_elasticsearch_serverless_branch.sh @@ -21,6 +21,10 @@ # Optional override: # ES_SERVERLESS_BRANCH — force this branch name # +# Resolution runs in sequential steps (no nested if/elif chain) so a fork-only +# branch warning never blocks later fallbacks to elastic/ (PR_SOURCE, then +# PR_TARGET on elastic). +# # Call pickElasticsearchServerlessBranch. It sets SERVERLESS_BRANCH and writes # diagnostics to stderr (callers often pipe stdout to buildkite-agent). @@ -35,6 +39,7 @@ function isElasticsearchServerlessBranchAtRemote { function pickElasticsearchServerlessBranch { SERVERLESS_BRANCH="main" + # 1) Explicit override (must exist on elastic/) if [ -n "${ES_SERVERLESS_BRANCH:-}" ]; then if isElasticsearchServerlessBranchAtRemote "elastic" "${ES_SERVERLESS_BRANCH}"; then SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH}" @@ -47,19 +52,24 @@ function pickElasticsearchServerlessBranch { return 1 fi - if [ -n "$PR_AUTHOR_FORK" ] && isElasticsearchServerlessBranchAtRemote "$PR_AUTHOR_FORK" "$PR_SOURCE"; then - if isElasticsearchServerlessBranchAtRemote "elastic" "$PR_SOURCE"; then - SERVERLESS_BRANCH="$PR_SOURCE" + # 2) Prefer PR_SOURCE when it exists on elastic/ (Buildkite only consumes elastic/) + if isElasticsearchServerlessBranchAtRemote "elastic" "$PR_SOURCE"; then + SERVERLESS_BRANCH="$PR_SOURCE" + if [ -n "$PR_AUTHOR_FORK" ] && isElasticsearchServerlessBranchAtRemote "$PR_AUTHOR_FORK" "$PR_SOURCE"; then echo "Found '$PR_SOURCE' on both $PR_AUTHOR_FORK and elastic; using elastic/" >&2 - else - echo "WARNING: Found '$PR_SOURCE' on $PR_AUTHOR_FORK/elasticsearch-serverless but not on elastic/." >&2 - echo "The trigger step can only use branches on elastic/elasticsearch-serverless." >&2 - echo "Push the branch to elastic/ or set ES_SERVERLESS_BRANCH explicitly." >&2 fi - elif isElasticsearchServerlessBranchAtRemote "elastic" "$PR_SOURCE"; then - SERVERLESS_BRANCH="$PR_SOURCE" - elif [ "$PR_TARGET" != "main" ] && isElasticsearchServerlessBranchAtRemote "elastic" "$PR_TARGET"; then - SERVERLESS_BRANCH="$PR_TARGET" + elif [ -n "$PR_AUTHOR_FORK" ] && isElasticsearchServerlessBranchAtRemote "$PR_AUTHOR_FORK" "$PR_SOURCE"; then + echo "WARNING: Found '$PR_SOURCE' on $PR_AUTHOR_FORK/elasticsearch-serverless but not on elastic/." >&2 + echo "The trigger step can only use branches on elastic/elasticsearch-serverless." >&2 + echo "Push the branch to elastic/ or set ES_SERVERLESS_BRANCH explicitly." >&2 + fi + + # 3) Still unresolved: fall back to PR base branch on elastic/ when available + if [ "$SERVERLESS_BRANCH" = "main" ] && [ -n "${PR_TARGET:-}" ] && [ "$PR_TARGET" != "main" ]; then + if isElasticsearchServerlessBranchAtRemote "elastic" "$PR_TARGET"; then + SERVERLESS_BRANCH="$PR_TARGET" + echo "Using elasticsearch-serverless branch '$PR_TARGET' from PR base (elastic/) as fallback." >&2 + fi fi echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2