diff --git a/.buildkite/ml_pipeline/config.py b/.buildkite/ml_pipeline/config.py index 7abb4a5371..8b13ec39d3 100644 --- a/.buildkite/ml_pipeline/config.py +++ b/.buildkite/ml_pipeline/config.py @@ -11,6 +11,17 @@ import os import re +# Keys allowed in the optional tail of trigger_comment_regex (group serverless_kv). +_SERVERLESS_KV_KEYS = frozenset( + { + "KEEP_DEPLOYMENT", + "REGION_ID", + "PROJECT_TYPE", + "ES_SERVERLESS_BRANCH", + } +) + + class Config: build_windows: bool = False build_macos: bool = False @@ -19,6 +30,8 @@ class Config: build_x86_64: str = "" run_qa_tests: bool = False run_pytorch_tests: bool = False + run_serverless_tests: bool = False + deploy_serverless_qa: bool = False action: str = "build" def parse_comment(self): @@ -37,9 +50,13 @@ def parse_comment(self): self.action = os.environ["GITHUB_PR_COMMENT_VAR_ACTION"] self.run_qa_tests = self.action == "run_qa_tests" self.run_pytorch_tests = self.action == "run_pytorch_tests" - if self.run_pytorch_tests or self.run_qa_tests: + self.run_serverless_tests = self.action == "run_serverless_tests" + self.deploy_serverless_qa = self.action == "deploy_serverless_qa" + if self.run_pytorch_tests or self.run_qa_tests or self.run_serverless_tests or self.deploy_serverless_qa: self.action = "build" + self._apply_serverless_kv_from_comment() + # If the ACTION is set to "run_qa_tests" then set some optional variables governing the ES branch to build, the # stack version to set and the subset of QA tests to run, depending on whether appropriate variables are set in # the environment. @@ -66,6 +83,9 @@ def parse_comment(self): self.build_x86_64 = "--build-x86_64" elif self.run_qa_tests or self.run_pytorch_tests: self.build_x86_64 = "--build-x86_64" + elif self.run_serverless_tests or self.deploy_serverless_qa: + self.build_aarch64 = "--build-aarch64" + self.build_x86_64 = "--build-x86_64" else: self.build_aarch64 = "--build-aarch64" self.build_x86_64 = "--build-x86_64" @@ -83,13 +103,21 @@ def parse_comment(self): self.build_macos = True elif each == "linux": self.build_linux = True - elif self.run_qa_tests or self.run_pytorch_tests: + elif self.run_qa_tests or self.run_pytorch_tests or self.run_serverless_tests or self.deploy_serverless_qa: self.build_linux = True else: self.build_windows = True self.build_macos = True self.build_linux = True + # Serverless runner pipelines depend on both Linux aarch64 and x86_64 + # build steps. Normalize after platform/arch parsing so PR comment tails + # cannot leave dangling depends_on keys or skip Linux builds. + if self.run_serverless_tests or self.deploy_serverless_qa: + self.build_aarch64 = "--build-aarch64" + self.build_x86_64 = "--build-x86_64" + self.build_linux = True + # If no explicit action was set (e.g. "buildkite test this" via # always_trigger_comment_regex), check PR labels for QA/PyTorch # flags. This is done after platform/arch defaults so that @@ -100,11 +128,15 @@ def parse_comment(self): self.run_qa_tests = True if "ci:run-pytorch-tests" in labels: self.run_pytorch_tests = True + if "ci:run-serverless-tests" in labels: + self.run_serverless_tests = True + if "ci:deploy-serverless-qa" in labels: + self.deploy_serverless_qa = True def parse_label(self): """ Parse labels set on GitHub PR comments.""" - build_labels = ['ci:build-linux','ci:build-macos','ci:build-windows','ci:run-qa-tests','ci:run-pytorch-tests','ci:build-aarch64','ci:build-x86_64'] + build_labels = ['ci:build-linux','ci:build-macos','ci:build-windows','ci:run-qa-tests','ci:run-pytorch-tests','ci:run-serverless-tests','ci:deploy-serverless-qa','ci:build-aarch64','ci:build-x86_64'] all_labels = [x.strip().lower() for x in os.environ["GITHUB_PR_LABELS"].split(",")] ci_labels = [label for label in all_labels if re.search("|".join(build_labels), label)] if not ci_labels: @@ -137,6 +169,16 @@ def parse_label(self): self.build_macos = True self.build_linux = True self.run_pytorch_tests = True + if "ci:run-serverless-tests" == label: + self.build_linux = True + self.build_aarch64 = "--build-aarch64" + self.build_x86_64 = "--build-x86_64" + self.run_serverless_tests = True + if "ci:deploy-serverless-qa" == label: + self.build_linux = True + self.build_aarch64 = "--build-aarch64" + self.build_x86_64 = "--build-x86_64" + self.deploy_serverless_qa = True if self.build_aarch64 == "" and self.build_x86_64 == "": self.build_aarch64 = "--build-aarch64" self.build_x86_64 = "--build-x86_64" @@ -156,3 +198,24 @@ def parse(self): self.build_x86_64 = "--build-x86_64" self.run_qa_tests = False + def _apply_serverless_kv_from_comment(self): + """Copy whitelisted KEY=value tokens from the PR comment regex capture into os.environ.""" + + env_key = "GITHUB_PR_COMMENT_VAR_SERVERLESS_KV" + if env_key not in os.environ: + return + raw = os.environ[env_key].strip() + if not raw: + return + for token in raw.split(): + key, sep, value = token.partition("=") + if not sep or key not in _SERVERLESS_KV_KEYS: + continue + if key == "KEEP_DEPLOYMENT" and value.lower() not in ("true", "false"): + continue + if key in ("REGION_ID", "PROJECT_TYPE") and not re.fullmatch(r"[A-Za-z0-9_.:-]+", value): + continue + if key == "ES_SERVERLESS_BRANCH" and not re.fullmatch(r"[A-Za-z0-9_./-]+", value): + continue + os.environ[key] = value + diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 0ae5776853..eff889d8a5 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -17,6 +17,7 @@ # import json +import os from ml_pipeline import ( step, @@ -52,6 +53,16 @@ def main(): "VERSION_QUALIFIER": "", "ML_BUILD_STEP_KEYS": ",".join(build_step_keys), } + if config.run_serverless_tests or config.deploy_serverless_qa: + for serverless_env_key in ( + "KEEP_DEPLOYMENT", + "REGION_ID", + "PROJECT_TYPE", + "ES_SERVERLESS_BRANCH", + ): + value = os.environ.get(serverless_env_key) + if value: + env[serverless_env_key] = value if config.build_windows: build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64) @@ -79,6 +90,17 @@ def main(): pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline", ".buildkite/pipelines/run_es_tests_aarch64.yml.sh")) + # Serverless tests/deploy require both Linux aarch64 and x86_64 build steps. + linux_both_arches = ( + config.build_linux and config.build_aarch64 and config.build_x86_64 + ) + if linux_both_arches and config.run_serverless_tests: + pipeline_steps.append(pipeline_steps.generate_step("Upload serverless tests runner pipeline", + ".buildkite/pipelines/run_serverless_tests.yml.sh")) + if linux_both_arches and config.deploy_serverless_qa: + pipeline_steps.append(pipeline_steps.generate_step("Upload serverless QA deploy pipeline", + ".buildkite/pipelines/deploy_serverless_qa.yml.sh")) + # Check for build timing regressions against nightly baseline pipeline_steps.append(pipeline_steps.generate_step("Check build timing regressions", ".buildkite/pipelines/check_build_regression.yml.sh", diff --git a/.buildkite/pipelines/deploy_serverless_qa.yml.sh b/.buildkite/pipelines/deploy_serverless_qa.yml.sh new file mode 100755 index 0000000000..c3d0eea318 --- /dev/null +++ b/.buildkite/pipelines/deploy_serverless_qa.yml.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +# Pipeline: build a serverless Docker image with custom ml-cpp and deploy it +# to the QA environment for interactive use. Unlike run_serverless_tests.yml.sh, +# this does NOT run E2E tests -- it just gets the environment running so the +# developer can interact with it (deploy models, run queries, kubectl, etc.). +# +# The deployment stays up for 1 hour by default. Set KEEP_DEPLOYMENT=true +# (via the Buildkite UI) to keep it longer. The build annotations will +# contain the URL and encrypted credentials for accessing the deployment. + +ML_CPP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=dev-tools/serverless_buildkite_trigger_prepare.sh +source "${ML_CPP_ROOT}/dev-tools/serverless_buildkite_trigger_prepare.sh" + +prepareMlCppServerlessTriggerContext "${BASH_SOURCE[0]}" || exit 1 +assignServerlessQaTriggerEnvYamlEscapes + +echo "Deploying to serverless QA with custom ml-cpp from PR #${PR_NUM}" >&2 + +cat </dev/null \ + | python3 -c "import sys,json; prs=json.load(sys.stdin); print(prs[0]['number'] if prs else '')" 2>/dev/null || true) + fi + if [ -z "$ES_PR_NUM" ]; then + ES_PR_NUM=$(curl -s "https://api.github.com/repos/elastic/elasticsearch/pulls?state=open&per_page=1" 2>/dev/null \ + | python3 -c "import sys,json; prs=json.load(sys.stdin); print(prs[0]['number'] if prs else '')" 2>/dev/null || true) + fi +fi +ES_PR_NUM="${ELASTICSEARCH_PR_NUMBER:-${ES_PR_NUM}}" +if [ -z "$ES_PR_NUM" ]; then + echo "WARNING: Could not resolve an ES PR number. The serverless PR-specific tests step may fail." >&2 +fi +echo "Using ES submodule commit: $ES_COMMIT, ES PR number: $ES_PR_NUM" >&2 + +assignServerlessQaTriggerEnvYamlEscapes + +cat <build|debug|run_qa_tests|run_pytorch_tests)(=(?(?:[^ ]+)))? *(?: for ES_BRANCH=(?([.0-9a-zA-Z]+)))? *(?:with STACK_VERSION=(?([.0-9]+)))? *(?: *on *(?(?:[ ,]*(?:windows|linux|mac(os)?))+))?) *(?(?:[, ]*aarch64|x86_64)+)?$", + "trigger_comment_regex": "^(?:(?:buildkite +)(?build|debug|run_qa_tests|run_pytorch_tests|run_serverless_tests|deploy_serverless_qa)(=(?(?:[^ ]+)))? *(?: for ES_BRANCH=(?([.0-9a-zA-Z]+)))? *(?:with STACK_VERSION=(?([.0-9]+)))? *(?: *on *(?(?:[ ,]*(?:windows|linux|mac(os)?))+))?) *(?(?:[, ]*aarch64|x86_64)+)?(?: *(?(?:(?:KEEP_DEPLOYMENT|REGION_ID|PROJECT_TYPE|ES_SERVERLESS_BRANCH)=[^\\s]+)(?: +(?:KEEP_DEPLOYMENT|REGION_ID|PROJECT_TYPE|ES_SERVERLESS_BRANCH)=[^\\s]+)*))?$", "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", "skip_ci_labels": ["skip-ci", "jenkins-ci", ">test-mute", ">docs"], "skip_target_branches": ["6.8", "7.11", "7.12"], diff --git a/dev-tools/pick_elasticsearch_clone_target.sh b/dev-tools/pick_elasticsearch_clone_target.sh new file mode 100644 index 0000000000..e1993625d9 --- /dev/null +++ b/dev-tools/pick_elasticsearch_clone_target.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# + +# Shared logic to choose which elasticsearch fork/branch to use for ml-cpp CI: +# integration test clones (run_es_tests_common.sh) and Buildkite pipelines that +# need ELASTICSEARCH_SUBMODULE_COMMIT without cloning. +# +# Source this file, then call pickCloneTarget. It reads (in order of precedence): +# GITHUB_PR_OWNER / GITHUB_PR_BRANCH — when the job is tied to a GitHub PR +# PR_AUTHOR / PR_SOURCE_BRANCH — fork and branch for coordinated ml-cpp + ES changes +# elastic / PR_SOURCE_BRANCH — upstream branch matching the ml-cpp PR branch name +# elastic / PR_TARGET_BRANCH — target branch of the ml-cpp PR +# elastic / main — final fallback +# +# On success, SELECTED_FORK and SELECTED_BRANCH are set. Optional helper +# elasticsearch_selected_branch_head_sha prints the remote HEAD commit for that +# pair (same transport as isCloneTargetValid: git@github.com). +# +# This file must be sourced (not executed) so that SELECTED_* remain in the caller's shell. + +function isCloneTargetValid { + local fork_to_check="$1" + local branch_to_check="$2" + if [ -z "$fork_to_check" ] || [ -z "$branch_to_check" ]; then + return 1 + fi + # Diagnostics must go to stderr: callers (e.g. deploy_serverless_qa.yml.sh) + # pipe stdout to `buildkite-agent pipeline upload` and expect only YAML. + echo "Checking for '$branch_to_check' branch at $fork_to_check/elasticsearch" >&2 + if [ -n "$(git ls-remote --heads "git@github.com:${fork_to_check}/elasticsearch.git" "$branch_to_check" 2>/dev/null)" ]; then + echo "Will use '$branch_to_check' branch at $fork_to_check/elasticsearch for ES integration tests" >&2 + return 0 + fi + return 1 +} + +SELECTED_FORK=elastic +SELECTED_BRANCH=main + +function pickCloneTarget { + + if isCloneTargetValid "$GITHUB_PR_OWNER" "$GITHUB_PR_BRANCH" ; then + SELECTED_FORK="$GITHUB_PR_OWNER" + SELECTED_BRANCH="$GITHUB_PR_BRANCH" + return 0 + fi + + if isCloneTargetValid "$PR_AUTHOR" "$PR_SOURCE_BRANCH" ; then + SELECTED_FORK="$PR_AUTHOR" + SELECTED_BRANCH="$PR_SOURCE_BRANCH" + return 0 + fi + + if isCloneTargetValid "$SELECTED_FORK" "$PR_SOURCE_BRANCH" ; then + SELECTED_BRANCH="$PR_SOURCE_BRANCH" + return 0 + fi + + if isCloneTargetValid "$SELECTED_FORK" "$PR_TARGET_BRANCH" ; then + SELECTED_BRANCH="$PR_TARGET_BRANCH" + return 0 + fi + + if isCloneTargetValid "$SELECTED_FORK" "$SELECTED_BRANCH" ; then + return 0 + fi + + return 1 +} + +# Prints the commit SHA at the head of SELECTED_BRANCH on SELECTED_FORK, or empty if unavailable. +function elasticsearch_selected_branch_head_sha { + git ls-remote --heads "git@github.com:${SELECTED_FORK}/elasticsearch.git" "${SELECTED_BRANCH}" 2>/dev/null | awk '{print $1; exit}' +} diff --git a/dev-tools/pick_elasticsearch_serverless_branch.sh b/dev-tools/pick_elasticsearch_serverless_branch.sh new file mode 100644 index 0000000000..5b3dcff418 --- /dev/null +++ b/dev-tools/pick_elasticsearch_serverless_branch.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# + +# Choose which branch of elastic/elasticsearch-serverless to pass to Buildkite +# trigger steps (es-pr-check, deploy-qa). This mirrors the fork/branch *idea* +# behind pick_elasticsearch_clone_target.sh, but for the serverless repo. +# +# Source this file after setting (from ml-cpp PR metadata): +# PR_AUTHOR_FORK — fork owner from BUILDKITE_BRANCH (author before ':') +# PR_SOURCE — branch name from BUILDKITE_BRANCH (after ':') +# PR_TARGET — BUILDKITE_PULL_REQUEST_BASE_BRANCH (default main) +# Optional override: +# ES_SERVERLESS_BRANCH — force this branch name +# +# Resolution runs in sequential steps (no nested if/elif chain) so a fork-only +# branch warning never blocks later fallbacks to elastic/ (PR_SOURCE, then +# PR_TARGET on elastic). +# +# Call pickElasticsearchServerlessBranch. It sets SERVERLESS_BRANCH and writes +# diagnostics to stderr (callers often pipe stdout to buildkite-agent). + +SERVERLESS_BRANCH="main" + +function isElasticsearchServerlessBranchAtRemote { + local repo="$1" + local branch="$2" + [ -n "$branch" ] && git ls-remote --heads "https://github.com/${repo}/elasticsearch-serverless.git" "$branch" 2>/dev/null | grep -q . +} + +function pickElasticsearchServerlessBranch { + SERVERLESS_BRANCH="main" + + # 1) Explicit override (must exist on elastic/) + if [ -n "${ES_SERVERLESS_BRANCH:-}" ]; then + if isElasticsearchServerlessBranchAtRemote "elastic" "${ES_SERVERLESS_BRANCH}"; then + SERVERLESS_BRANCH="${ES_SERVERLESS_BRANCH}" + echo "Using explicit ES_SERVERLESS_BRANCH override: $SERVERLESS_BRANCH" >&2 + echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 + return 0 + fi + echo "ERROR: ES_SERVERLESS_BRANCH override '${ES_SERVERLESS_BRANCH}' was not found on elastic/elasticsearch-serverless." >&2 + echo "Set ES_SERVERLESS_BRANCH to an existing branch on elastic/elasticsearch-serverless." >&2 + return 1 + fi + + # 2) Prefer PR_SOURCE when it exists on elastic/ (Buildkite only consumes elastic/) + if isElasticsearchServerlessBranchAtRemote "elastic" "$PR_SOURCE"; then + SERVERLESS_BRANCH="$PR_SOURCE" + if [ -n "$PR_AUTHOR_FORK" ] && isElasticsearchServerlessBranchAtRemote "$PR_AUTHOR_FORK" "$PR_SOURCE"; then + echo "Found '$PR_SOURCE' on both $PR_AUTHOR_FORK and elastic; using elastic/" >&2 + fi + elif [ -n "$PR_AUTHOR_FORK" ] && isElasticsearchServerlessBranchAtRemote "$PR_AUTHOR_FORK" "$PR_SOURCE"; then + echo "WARNING: Found '$PR_SOURCE' on $PR_AUTHOR_FORK/elasticsearch-serverless but not on elastic/." >&2 + echo "The trigger step can only use branches on elastic/elasticsearch-serverless." >&2 + echo "Push the branch to elastic/ or set ES_SERVERLESS_BRANCH explicitly." >&2 + fi + + # 3) Still unresolved: fall back to PR base branch on elastic/ when available + if [ "$SERVERLESS_BRANCH" = "main" ] && [ -n "${PR_TARGET:-}" ] && [ "$PR_TARGET" != "main" ]; then + if isElasticsearchServerlessBranchAtRemote "elastic" "$PR_TARGET"; then + SERVERLESS_BRANCH="$PR_TARGET" + echo "Using elasticsearch-serverless branch '$PR_TARGET' from PR base (elastic/) as fallback." >&2 + fi + fi + + echo "Resolved elasticsearch-serverless branch: $SERVERLESS_BRANCH" >&2 +} diff --git a/dev-tools/run_es_tests_common.sh b/dev-tools/run_es_tests_common.sh index 9dcb5f8509..4419a3c013 100755 --- a/dev-tools/run_es_tests_common.sh +++ b/dev-tools/run_es_tests_common.sh @@ -33,50 +33,8 @@ set -e -function isCloneTargetValid { - FORK_TO_CHECK="$1" - BRANCH_TO_CHECK="$2" - echo "Checking for '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch" - if [ -n "$(git ls-remote --heads "git@github.com:$FORK_TO_CHECK/elasticsearch.git" "$BRANCH_TO_CHECK" 2>/dev/null)" ]; then - echo "Will use '$BRANCH_TO_CHECK' branch at $FORK_TO_CHECK/elasticsearch for ES integration tests" - return 0 - fi - return 1 -} - -SELECTED_FORK=elastic -SELECTED_BRANCH=main - -function pickCloneTarget { - - if isCloneTargetValid "$GITHUB_PR_OWNER" "$GITHUB_PR_BRANCH" ; then - SELECTED_FORK="$GITHUB_PR_OWNER" - SELECTED_BRANCH="$GITHUB_PR_BRANCH" - return 0 - fi - - if isCloneTargetValid "$PR_AUTHOR" "$PR_SOURCE_BRANCH" ; then - SELECTED_FORK="$PR_AUTHOR" - SELECTED_BRANCH="$PR_SOURCE_BRANCH" - return 0 - fi - - if isCloneTargetValid "$SELECTED_FORK" "$PR_SOURCE_BRANCH" ; then - SELECTED_BRANCH="$PR_SOURCE_BRANCH" - return 0 - fi - - if isCloneTargetValid "$SELECTED_FORK" "$PR_TARGET_BRANCH" ; then - SELECTED_BRANCH="$PR_TARGET_BRANCH" - return 0 - fi - - if isCloneTargetValid "$SELECTED_FORK" "$SELECTED_BRANCH" ; then - return 0 - fi - - return 1 -} +# shellcheck source=pick_elasticsearch_clone_target.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/pick_elasticsearch_clone_target.sh" CLONE_DIR="$1" IVY_REPO_PATH="$2" diff --git a/dev-tools/serverless_buildkite_trigger_prepare.sh b/dev-tools/serverless_buildkite_trigger_prepare.sh new file mode 100644 index 0000000000..272c96935a --- /dev/null +++ b/dev-tools/serverless_buildkite_trigger_prepare.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# + +# Shared helpers for Buildkite pipeline scripts that trigger elasticsearch-serverless +# (deploy QA, es-pr-check). Source after setting ML_CPP_ROOT, then call: +# prepareMlCppServerlessTriggerContext "${BASH_SOURCE[0]}" || exit 1 +# assignServerlessQaTriggerEnvYamlEscapes +# Use $(emitServerlessUploadMlCppDepsStepYaml) inside a "steps:" heredoc body. + +function yamlDoubleQuoteEscape { + printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' +} + +function assignServerlessQaTriggerEnvYamlEscapes { + KEEP_DEPLOYMENT_SAFE=$(yamlDoubleQuoteEscape "${KEEP_DEPLOYMENT:-false}") + REGION_ID_SAFE=$(yamlDoubleQuoteEscape "${REGION_ID:-aws-eu-west-1}") + PROJECT_TYPE_SAFE=$(yamlDoubleQuoteEscape "${PROJECT_TYPE:-elasticsearch}") +} + +function prepareMlCppServerlessTriggerContext { + local pipeline_script="$1" + if [ -z "$pipeline_script" ]; then + echo "prepareMlCppServerlessTriggerContext: missing path to the calling pipeline script" >&2 + return 1 + fi + + SAFE_MESSAGE=$(printf '%s' "${BUILDKITE_MESSAGE}" | head -1 | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g') + PR_NUM="${BUILDKITE_PULL_REQUEST}" + if [ -z "${PR_NUM}" ] || [ "${PR_NUM}" = "false" ]; then + PR_NUM="manual" + fi + + PR_AUTHOR_FORK="$(expr "${BUILDKITE_BRANCH:-}" : '\(.*\):.*' 2>/dev/null || true)" + PR_SOURCE="$(expr "${BUILDKITE_BRANCH:-}" : '.*:\(.*\)' 2>/dev/null || true)" + PR_TARGET="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" + + ML_CPP_ROOT="$(cd "$(dirname "${pipeline_script}")/../.." && pwd)" + # shellcheck source=dev-tools/pick_elasticsearch_clone_target.sh + source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_clone_target.sh" + export PR_AUTHOR="${PR_AUTHOR_FORK}" + export PR_SOURCE_BRANCH="${PR_SOURCE}" + export PR_TARGET_BRANCH="${PR_TARGET}" + + # shellcheck source=dev-tools/pick_elasticsearch_serverless_branch.sh + source "${ML_CPP_ROOT}/dev-tools/pick_elasticsearch_serverless_branch.sh" + pickElasticsearchServerlessBranch || return 1 + + pickCloneTarget || true + ES_COMMIT="$(elasticsearch_selected_branch_head_sha)" + ES_COMMIT="${ES_COMMIT:-HEAD}" + echo "Resolved elasticsearch submodule: ${SELECTED_FORK}/${SELECTED_BRANCH} -> ${ES_COMMIT}" >&2 +} + +function emitServerlessUploadMlCppDepsStepYaml { + cat <<'EOS' + - label: ":package: Upload ml-cpp deps artifact" + key: "upload_ml_cpp_deps" + command: 'buildkite-agent artifact upload dev-tools/minimal.zip' + depends_on: + - "build_test_linux-x86_64-RelWithDebInfo" + - "build_test_linux-aarch64-RelWithDebInfo" + agents: + provider: aws + instanceType: m6i.xlarge + imagePrefix: core-amazonlinux-2023 + diskSizeGb: 100 + diskName: '/dev/xvda' + +EOS +} +