Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/grype_scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ jobs:
./.github/grype/run_grype_scan.sh $DOCKER_IMAGE

- name: Parse grype results
id: parse_grype
run: |
python3 -u ./.github/grype/parse_vulnerabilities_grype.py -o nice --no-colors --log raw.log --test-to-end

Expand Down Expand Up @@ -138,7 +139,7 @@ jobs:
owner: context.repo.owner,
repo: context.repo.repo,
sha: '${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}',
state: hasError ? 'error' : hasVulnerabilities ? 'failure' : 'success',
state: hasError ? 'error' : '${{ steps.parse_grype.outcome == 'success' && 'success' || 'failure' }}',
target_url: '${{ steps.upload_results.outputs.https_report_path }}',
description: hasError ? 'An error occurred' : `Grype Scan Completed with ${totalHighCritical} high/critical vulnerabilities`,
context: 'Grype Scan ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}'
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/master.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4182,7 +4182,7 @@ jobs:
commit: 979bb27171f92724bcd8f086989ba623f2e03fdc
arch: release
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout_minutes: 300
timeout_minutes: 210
workflow_config: ${{ needs.config_workflow.outputs.data }}
RegressionTestsAarch64:
needs: [config_workflow, build_arm_binary]
Expand All @@ -4194,7 +4194,7 @@ jobs:
commit: 979bb27171f92724bcd8f086989ba623f2e03fdc
arch: aarch64
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout_minutes: 300
timeout_minutes: 210
workflow_config: ${{ needs.config_workflow.outputs.data }}

SignRelease:
Expand Down
26 changes: 13 additions & 13 deletions .github/workflows/pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ jobs:

build_amd_debug:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9kZWJ1Zyk=') }}
name: "Build (amd_debug)"
outputs:
Expand Down Expand Up @@ -323,7 +323,7 @@ jobs:

build_amd_release:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
name: "Build (amd_release)"
outputs:
Expand Down Expand Up @@ -368,7 +368,7 @@ jobs:

build_amd_asan:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9hc2FuKQ==') }}
name: "Build (amd_asan)"
outputs:
Expand Down Expand Up @@ -413,7 +413,7 @@ jobs:

build_amd_tsan:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF90c2FuKQ==') }}
name: "Build (amd_tsan)"
outputs:
Expand Down Expand Up @@ -458,7 +458,7 @@ jobs:

build_amd_msan:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9tc2FuKQ==') }}
name: "Build (amd_msan)"
outputs:
Expand Down Expand Up @@ -503,7 +503,7 @@ jobs:

build_amd_ubsan:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF91YnNhbik=') }}
name: "Build (amd_ubsan)"
outputs:
Expand Down Expand Up @@ -548,7 +548,7 @@ jobs:

build_amd_binary:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9iaW5hcnkp') }}
name: "Build (amd_binary)"
outputs:
Expand Down Expand Up @@ -593,7 +593,7 @@ jobs:

build_arm_release:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
name: "Build (arm_release)"
outputs:
Expand Down Expand Up @@ -638,7 +638,7 @@ jobs:

build_arm_coverage:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9jb3ZlcmFnZSk=') }}
name: "Build (arm_coverage)"
outputs:
Expand Down Expand Up @@ -683,7 +683,7 @@ jobs:

build_arm_binary:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9iaW5hcnkp') }}
name: "Build (arm_binary)"
outputs:
Expand Down Expand Up @@ -2167,7 +2167,7 @@ jobs:
fi

stateless_tests_amd_asan_flaky_check:
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_amd_asan]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbiwgZmxha3kgY2hlY2sp') }}
name: "Stateless tests (amd_asan, flaky check)"
Expand Down Expand Up @@ -4047,7 +4047,7 @@ jobs:
commit: 979bb27171f92724bcd8f086989ba623f2e03fdc
arch: release
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout_minutes: 300
timeout_minutes: 210
workflow_config: ${{ needs.config_workflow.outputs.data }}
RegressionTestsAarch64:
needs: [config_workflow, build_arm_binary]
Expand All @@ -4059,7 +4059,7 @@ jobs:
commit: 979bb27171f92724bcd8f086989ba623f2e03fdc
arch: aarch64
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout_minutes: 300
timeout_minutes: 210
workflow_config: ${{ needs.config_workflow.outputs.data }}

FinishCIReport:
Expand Down
11 changes: 6 additions & 5 deletions .github/workflows/regression-reusable-suite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ jobs:
SUITE_EXECUTABLE: ${{ inputs.suite_executable }}
STORAGE: ${{ inputs.storage_path }}
PART: ${{ inputs.part }}
REPORT_JOB_NAME: ${{ format('{0}{1}', inputs.job_name != '' && inputs.job_name || inputs.suite_name, inputs.part != '' && format('_{0}', inputs.part) || '') }}
# AWS credentials
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
Expand All @@ -98,7 +99,7 @@ jobs:
--local
--collect-service-logs
--output ${{ inputs.output_format }}
--attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" user.name="${GITHUB_ACTOR}" version="${{ fromJson(inputs.workflow_config).custom_data.version.string }}" package="$clickhouse_path" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
--attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" user.name="${GITHUB_ACTOR}" version="${{ fromJson(inputs.workflow_config).custom_data.version.string }}" package="$clickhouse_path" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$REPORT_JOB_NAME" job.retry=$GITHUB_RUN_ATTEMPT job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
--cicd
--log raw.log
${{ inputs.flags != 'none' && inputs.flags || ''}}
Expand Down Expand Up @@ -168,24 +169,24 @@ jobs:
exit $EXITCODE

- name: 📊 Set Commit Status
if: always() && inputs.set_commit_status
if: ${{ !cancelled() && inputs.set_commit_status }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
JOB_OUTCOME: ${{ steps.run_suite.outcome }}
SUITE_NAME: ${{ format('Regression {0} {1}{2}', inputs.runner_arch, inputs.job_name != '' && inputs.job_name || inputs.suite_name, inputs.part != '' && format('_{0}', inputs.part) || '') }}
run: python3 .github/set_builds_status.py

- name: 📝 Create and upload logs
if: always()
if: ${{ !cancelled() }}
run: .github/create_and_upload_logs.sh 1

- name: 📤 Upload logs to results database
if: always()
if: ${{ !cancelled() }}
timeout-minutes: 20
run: .github/upload_results_to_database.sh 1

- uses: actions/upload-artifact@v4
if: always()
if: ${{ !cancelled() }}
with:
name: ${{ format('{0}{1}-artifacts-{2}{3}', inputs.job_name != '' && inputs.job_name || inputs.suite_name, inputs.part != '' && format('_{0}', inputs.part) || '', inputs.runner_arch, contains(inputs.extra_args, '--use-keeper') && '_keeper' || '_zookeeper') }}
path: ${{ env.artifact_paths }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/regression.yml
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ jobs:
strategy:
fail-fast: false
matrix:
SUITE: [aes_encryption, atomic_insert, base_58, data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, functions, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, selects, session_timezone, swarms, version, window_functions]
SUITE: [aes_encryption, atomic_insert, attach, base_58, clickhouse_keeper_failover,data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, functions, jwt_authentication, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, selects, session_timezone, settings, swarms, version, window_functions]
uses: ./.github/workflows/regression-reusable-suite.yml
with:
ref: ${{ inputs.commit }}
Expand Down Expand Up @@ -346,7 +346,7 @@ jobs:
build_sha: ${{ inputs.build_sha }}
set_commit_status: true
job_name: parquet_${{ matrix.STORAGE }}
regression_args: --storage ${{ matrix.STORAGE }} --aws-s3-bucket {{AWS_BUCKET}} --aws-s3-region {{AWS_REGION}} --aws-s3-key-id {{AWS_KEY_ID}} --aws-s3-access-key {{AWS_ACCESS_KEY}}
regression_args: --storage ${{ matrix.STORAGE }} --aws-s3-bucket {{AWS_BUCKET}} --aws-s3-region {{AWS_REGION}} --aws-s3-key-id {{AWS_KEY_ID}} --aws-s3-access-key {{AWS_ACCESS_KEY}} --only "/parquet/${{ matrix.STORAGE }}/*"
secrets: inherit

RBAC:
Expand Down
2 changes: 1 addition & 1 deletion ci/defs/job_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,7 @@ class JobConfigs:
stateless_tests_flaky_pr_jobs = common_ft_job_config.parametrize(
Job.ParamSet(
parameter="amd_asan, flaky check",
runs_on=RunnerLabels.AMD_SMALL_MEM,
runs_on=RunnerLabels.FUNC_TESTER_AMD,
requires=[ArtifactNames.CH_AMD_ASAN],
),
)
Expand Down
2 changes: 1 addition & 1 deletion ci/jobs/scripts/clickhouse_proc.py
Original file line number Diff line number Diff line change
Expand Up @@ -790,7 +790,7 @@ def check_fatal_messages_in_logs(self):
results.append(
Result.from_commands_run(
name="Logical error thrown (in clickhouse-server.log or clickhouse-server.err.log)",
command=f"cd {self.log_dir} && ! grep -a -A10 'Code: 49. DB::Exception: ' clickhouse-server*.log | head -n100 | tee /dev/stderr | grep -q .",
command=f"cd {self.log_dir} && ! grep -a -A10 -e 'Code: 49. DB::Exception: ' -e '<Fatal> : Logical error:' clickhouse-server*.log | head -n100 | tee /dev/stderr | grep -q .",
)
)
results.append(
Expand Down
4 changes: 2 additions & 2 deletions ci/praktika/yaml_additional_templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ class AltinityWorkflowTemplates:
commit: {REGRESSION_HASH}
arch: release
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout_minutes: 300
timeout_minutes: 210
workflow_config: ${{ needs.config_workflow.outputs.data }}
RegressionTestsAarch64:
needs: [config_workflow, build_arm_binary]
Expand All @@ -83,7 +83,7 @@ class AltinityWorkflowTemplates:
commit: {REGRESSION_HASH}
arch: aarch64
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout_minutes: 300
timeout_minutes: 210
workflow_config: ${{ needs.config_workflow.outputs.data }}
""",
"SignRelease": r"""
Expand Down
2 changes: 1 addition & 1 deletion ci/workflows/pull_request.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
job.set_dependency(
[
# JobNames.STYLE_CHECK, # NOTE (strtgbb): we don't run style check
# JobNames.FAST_TEST, # NOTE (strtgbb): this takes too long, revisit later
JobNames.FAST_TEST,
# JobConfigs.tidy_build_arm_jobs[0].name, # NOTE (strtgbb): this takes too long, revisit later
]
)
Expand Down
14 changes: 3 additions & 11 deletions docker/keeper/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,8 @@ RUN arch=${TARGETARCH:-amd64} \
arm64) ln -sf /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1 ;; \
esac

# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
ARG VERSION="25.7.4.11"
ARG PACKAGES="clickhouse-keeper"
# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile

ARG DIRECT_DOWNLOAD_URLS=""

# user/group precreated explicitly with fixed uid/gid on purpose.
Expand All @@ -63,12 +60,7 @@ RUN arch=${TARGETARCH:-amd64} \
&& wget -c -q "$url" \
; done \
else \
for package in ${PACKAGES}; do \
cd /tmp \
&& echo "Get ${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
&& wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
&& wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz.sha512" \
; done \
exit 1; \
fi \
&& cat *.tgz.sha512 | sha512sum -c \
&& for file in *.tgz; do \
Expand Down
15 changes: 5 additions & 10 deletions docker/server/Dockerfile.alpine
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,10 @@ RUN arch=${TARGETARCH:-amd64} \
arm64) ln -sf /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1 ;; \
esac

# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
ARG VERSION="25.7.4.11"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"


# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile

ARG DIRECT_DOWNLOAD_URLS=""

# user/group precreated explicitly with fixed uid/gid on purpose.
Expand All @@ -59,11 +58,7 @@ RUN arch=${TARGETARCH:-amd64} \
&& wget -c -q "$url" \
; done \
else \
for package in ${PACKAGES}; do \
echo "Get ${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
&& wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
&& wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz.sha512" \
; done \
exit 1; \
fi \
&& cat *.tgz.sha512 | sed 's:/output/:/tmp/:' | sha512sum -c \
&& for file in *.tgz; do \
Expand Down
Loading
Loading