diff --git a/.bazelrc b/.bazelrc index 2c33ecf..5e2f91a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -58,10 +58,13 @@ build:engflow_bes --bes_results_url=https://opal.cluster.engflow.com/invocations build:engflow_bes --bes_instance_name=auth build:engflow_bes --bes_lifecycle_events -build:engflow --config=engflow_common -build:engflow --config=engflow_bes +build:engflow_cache --config=engflow_common +build:engflow_cache --config=engflow_bes +build:engflow_cache --remote_cache=grpcs://opal.cluster.engflow.com +build:engflow_cache --remote_instance_name=auth + +build:engflow --config=engflow_cache build:engflow --remote_executor=grpcs://opal.cluster.engflow.com -build:engflow --remote_instance_name=auth # To authenticate with the clusters above, either add flags to this # .bazelrc.user file or to your $HOME/.bazelrc. These files may contain diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5e975f..90b31d1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,17 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Postsubmit checks that run on the `main` branch after merge. name: "main" on: workflow_dispatch: + pull_request: push: branches: - main concurrency: group: ${{ github.workflow }}.${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: # Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467 @@ -33,62 +34,30 @@ env: jobs: # TODO(OND-616): use remote execution and caching for all CI Runner jobs. - bazel-builder: - runs-on: - - self-hosted - - os=linux - - arch=x64 - - "engflow-cluster=glass" - - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0" - - "engflow-pool=ci_sysbox_x64" - - "engflow-runtime=sysbox-runc" - - "engflow-runner-id=${{ github.repository_id }}_bazel-builder_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" - - "engflow-run-id=${{ github.run_id }}" + build-and-test: + runs-on: ${{ matrix.runner }} timeout-minutes: 30 - - env: - ARCH: "x64" - OS: "linux" - - steps: - - uses: actions/checkout@v4 - - - name: Log in - run: infra/login.sh - - - name: Run all tests - if: success() - run: | - bazel test --config=noninteractive --config=engflow --config=remote_linux_x64 //... - - - name: Log out - run: infra/logout.sh - - golang-builder: - runs-on: - - self-hosted - - "purpose=github-actions" - - "os=${{ matrix.os }}" - - "arch=${{ matrix.arch }}" - - "os_distribution=${{ matrix.os_distribution }}" - - "os_version=${{ matrix.os_version }}" - - "revision=${{ matrix.revision }}" - timeout-minutes: 10 strategy: fail-fast: false matrix: + # Use remote execution on Linux, remote cache for others. macOS and Windows + # are sensitive to the C++ toolchain installed on the runner, and it may not + # match what's installed on the remote worker. include: + - os: "linux" + arch: "x64" + runner: "ubuntu-latest" + ef_config: "engflow" + - os: "macos" arch: "arm64" - os_distribution: "sonoma" - os_version: "14" - revision: "c12cc4655257fcf7da86fe06e87542b7e6814a1f" + runner: "macos-14" + ef_config: "engflow_cache" - os: "windows" arch: "x64" - os_distribution: "server" - os_version: "2022" - revision: "c12cc4655257fcf7da86fe06e87542b7e6814a1f" + runner: "windows-2022" + ef_config: "engflow_cache" env: ARCH: "${{ matrix.arch }}" @@ -97,66 +66,29 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Log in + - name: "Log in" shell: bash run: infra/login.sh - - name: Run all tests + - name: "Bazel tests" if: success() shell: bash run: | - # TODO(CUS-345): Enable remote execution - bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- test ./... - bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- clean -cache -modcache - - - name: Log out - shell: bash - run: infra/logout.sh - - golang-builder-ci-runners: - runs-on: - - self-hosted - - os=linux - - arch=x64 - - "engflow-cluster=glass" - - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0" - - "engflow-pool=ci_sysbox_x64" - - "engflow-runtime=sysbox-runc" - - "engflow-runner-id=${{ github.repository_id }}_golang-builder-ci-runners_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" - - "engflow-run-id=${{ github.run_id }}" - timeout-minutes: 10 - - env: - ARCH: "x64" - OS: "linux" - - steps: - - uses: actions/checkout@v4 + bazel test --config=noninteractive --config=${{ matrix.ef_config }} --config=remote_${{ matrix.os }}_${{ matrix.arch }} //... - - name: Log in - run: infra/login.sh - - - name: Run all tests + - name: "Go tests" if: success() + shell: bash run: | - # TODO(CUS-345): Enable remote execution bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- test ./... bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- clean -cache -modcache - name: Log out + shell: bash run: infra/logout.sh copyright-headers-check: - runs-on: - - self-hosted - - os=linux - - arch=x64 - - "engflow-cluster=glass" - - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0" - - "engflow-pool=ci_sysbox_x64" - - "engflow-runtime=sysbox-runc" - - "engflow-runner-id=${{ github.repository_id }}_copyright-headers-check_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" - - "engflow-run-id=${{ github.run_id }}" + runs-on: ubuntu-latest timeout-minutes: 10 env: diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml deleted file mode 100644 index 69305c1..0000000 --- a/.github/workflows/presubmit.yml +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2024 EngFlow Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Presubmit checks for PRs -name: "presubmit" - -on: - workflow_dispatch: - # Trigger on pull request rather than push, so that we can control whether - # checks are run on a given PR (allowing checks to run automatically on PR - # updates from third parties can be a security issue). - pull_request: - -concurrency: - group: ${{ github.workflow }}.${{ github.ref }} - cancel-in-progress: true - -env: - # Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467 - BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CLUSTER_HOST: opal.cluster.engflow.com - CRED_HELPER_TOKEN: ${{ secrets.OPAL_CRED_HELPER_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -jobs: - bazel-builder: - runs-on: - - self-hosted - - os=linux - - arch=x64 - - "engflow-cluster=glass" - - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0" - - "engflow-pool=ci_sysbox_x64" - - "engflow-runtime=sysbox-runc" - - "engflow-runner-id=${{ github.repository_id }}_bazel-builder_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" - - "engflow-run-id=${{ github.run_id }}" - timeout-minutes: 30 - - env: - ARCH: "x64" - OS: "linux" - - steps: - - uses: actions/checkout@v4 - - - name: Log in - run: infra/login.sh - - - name: Run all tests - if: success() - run: | - bazel test \ - --config=noninteractive \ - --config=engflow \ - --config=remote_linux_x64 \ - //... - - - name: Log out - run: infra/logout.sh - - golang-builder: - runs-on: - - self-hosted - - "purpose=github-actions" - - "os=${{ matrix.os }}" - - "arch=${{ matrix.arch }}" - - "os_distribution=${{ matrix.os_distribution }}" - - "os_version=${{ matrix.os_version }}" - - "revision=${{ matrix.revision }}" - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - include: - - os: "macos" - arch: "arm64" - os_distribution: "sonoma" - os_version: "14" - revision: "c12cc4655257fcf7da86fe06e87542b7e6814a1f" - - - os: "windows" - arch: "x64" - os_distribution: "server" - os_version: "2022" - revision: "c12cc4655257fcf7da86fe06e87542b7e6814a1f" - - env: - ARCH: "${{ matrix.arch }}" - OS: "${{ matrix.os }}" - - steps: - - uses: actions/checkout@v4 - - - name: Log in - shell: bash - run: infra/login.sh - - - name: Run all tests - if: success() - shell: bash - run: | - # TODO(CUS-345): Enable remote execution - bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- test ./... - bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- clean -cache -modcache - - - name: Log out - shell: bash - run: infra/logout.sh - - golang-builder-ci-runners: - runs-on: - - self-hosted - - os=linux - - arch=x64 - - "engflow-cluster=glass" - - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0" - - "engflow-pool=ci_sysbox_x64" - - "engflow-runtime=sysbox-runc" - - "engflow-runner-id=${{ github.repository_id }}_golang-builder-ci-runners_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" - - "engflow-run-id=${{ github.run_id }}" - timeout-minutes: 10 - - env: - ARCH: "x64" - OS: "linux" - - steps: - - uses: actions/checkout@v4 - - - name: Log in - run: infra/login.sh - - - name: Run all tests - if: success() - run: | - # TODO(CUS-345): Enable remote execution - bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- test ./... - bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- clean -cache -modcache - - - name: Log out - run: infra/logout.sh - - copyright-headers-check: - runs-on: - - self-hosted - - os=linux - - arch=x64 - - "engflow-cluster=glass" - - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0" - - "engflow-pool=ci_sysbox_x64" - - "engflow-runtime=sysbox-runc" - - "engflow-runner-id=${{ github.repository_id }}_copyright-headers-check_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" - - "engflow-run-id=${{ github.run_id }}" - timeout-minutes: 10 - - env: - ARCH: "x64" - OS: "linux" - - steps: - - uses: actions/checkout@v4 - - - name: Log in - run: infra/login.sh - - - name: Check copyright headers - if: success() - run: | - bazel run --config=noninteractive --config=engflow_bes //infra/internal/check_copyright_headers - - - name: Log out - run: infra/logout.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1f127e..2ceed60 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,72 +42,15 @@ env: jobs: check-version: - runs-on: - - self-hosted - - os=linux - - arch=x64 - - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0" - - "engflow-pool=ci_sysbox_x64" - - "engflow-runtime=sysbox-runc" - - "engflow-runner-id=${{ github.repository_id }}_check-version_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" + runs-on: ubuntu-latest timeout-minutes: 1 steps: - uses: actions/checkout@v4 - run: infra/release-check.sh - build-linux-artifacts: - # Linux is separate because we execute with CI runners, which don't support - # macOS or Windows yet. We use a Debian 11 image because binaries built on - # newer versions are incompatible with older version due to the libc runtime - # dependency. - runs-on: - - self-hosted - - os=linux - - arch=x64 - - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian11-dind-x64@sha256:604855f1ecd6edad3f45f513ed0112b9e8e32589871133767a1a51944b07f487" - - "engflow-pool=ci_sysbox_x64" - - "engflow-runtime=sysbox-runc" - - "engflow-runner-id=${{ github.repository_id }}_build-linux-artifacts_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" - timeout-minutes: 10 - needs: - - check-version - - env: - ARCH: "x64" - OS: "linux" - - steps: - - uses: actions/checkout@v4 - - - name: Log in - run: infra/login.sh - - - name: Build release artifacts - if: success() - run: infra/release-build.sh - - - uses: actions/upload-artifact@v4 - if: success() - with: - name: linux - path: _out - if-no-files-found: error - retention-days: 1 - overwrite: true - - - name: Log out - run: infra/logout.sh - - build-other-artifacts: - runs-on: - - self-hosted - - "purpose=github-actions" - - "os=${{ matrix.os }}" - - "arch=${{ matrix.arch }}" - - "os_distribution=${{ matrix.os_distribution }}" - - "os_version=${{ matrix.os_version }}" - - "revision=c12cc4655257fcf7da86fe06e87542b7e6814a1f" + build-artifacts: + runs-on: ${{ matrix.runner }} timeout-minutes: 10 needs: - check-version @@ -116,15 +59,17 @@ jobs: fail-fast: false matrix: include: + - os: "linux" + arch: "x64" + runner: "ubuntu-latest" + - os: "macos" arch: "arm64" - os_distribution: "sonoma" - os_version: "14" + runner: "macos-14" - os: "windows" arch: "x64" - os_distribution: "server" - os_version: "2022" + runner: "windows-2022" env: ARCH: ${{ matrix.arch }} @@ -156,18 +101,10 @@ jobs: shell: bash create-release: - runs-on: - - self-hosted - - os=linux - - arch=x64 - - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0" - - "engflow-pool=ci_sysbox_x64" - - "engflow-runtime=sysbox-runc" - - "engflow-runner-id=${{ github.repository_id }}_create-release_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" + runs-on: ubuntu-latest timeout-minutes: 10 needs: - - build-other-artifacts - - build-linux-artifacts + - build-artifacts steps: - uses: actions/checkout@v4