From 32aebbb18ff604966b5d75bcb90b6c4b06e90e90 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 14 Aug 2024 14:32:54 +0200 Subject: [PATCH 1/3] CI: adapt [check_commit_messages.sh] to non-merged result pipelines --- scripts/ci/check_commit_messages.sh | 30 +++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/scripts/ci/check_commit_messages.sh b/scripts/ci/check_commit_messages.sh index f78860a0a184..2ba16766f985 100755 --- a/scripts/ci/check_commit_messages.sh +++ b/scripts/ci/check_commit_messages.sh @@ -40,8 +40,34 @@ pattern_file="${script_dir}forbidden_commit_messages.txt" # Use git_merge_base.sh on merged result pipelines, to ensure we only # check commit messages of the MR in question. ORIGIN=${ORIGIN:-origin} -head=${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-HEAD} -target=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA:-${ORIGIN}/master} + +# CI_MERGE_REQUEST_{SOURCE,TARGET}_BRANCH_SHA are only available in +# merged result pipelines. In merge request pipelines, use +# CI_MERGE_REQUEST_TARGET_BRANCH_NAME instead. +if [ -n "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-}" ] && + [ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_SHA:-}" ]; then + head=${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA} + target=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA} +elif [ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-}" ]; then + head=HEAD + target=${ORIGIN}/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} + + # Make CI_MERGE_REQUEST_TARGET_BRANCH_NAME available locally -- this + # is a prerequisite for 'git_merge_base.sh' when it's operands are + # not fully spelled out hex object names (e.g. when they are branch + # names instead of commit SHAs). + git fetch --depth 1 "${ORIGIN}" "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" +else + car << EOT +None of: + - CI_MERGE_REQUEST_SOURCE_BRANCH_SHA + - CI_MERGE_REQUEST_TARGET_BRANCH_SHA + - CI_MERGE_REQUEST_TARGET_BRANCH_NAME +are set. Cannot retrieve the commit history of this MR. +EOT + exit 1 +fi + merge_base=$("$src_dir"/scripts/ci/git_merge_base.sh "$target" "$head" || { # Print on stderr to make visible outside command substitution echo "Failed to get merge base, cannot check commit messages." >&2 -- GitLab From 4ce26ef90262a3b5ce47a9fdfb72e08124ff8a8e Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 14 Aug 2024 14:57:06 +0200 Subject: [PATCH 2/3] CI: disallow [check_commit_messages.sh] failures --- .gitlab/ci/pipelines/before_merging.yml | 3 ++- .gitlab/ci/pipelines/merge_train.yml | 3 ++- ci/bin/code_verification.ml | 3 +-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 4a18219fe433..ffde65ba7b66 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -323,7 +323,8 @@ commit_titles: - trigger dependencies: - oc.docker:ci:amd64 - allow_failure: true + allow_failure: + exit_codes: 65 timeout: 60 minutes script: - ./scripts/ci/check_commit_messages.sh || exit $? diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 4a18219fe433..ffde65ba7b66 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -323,7 +323,8 @@ commit_titles: - trigger dependencies: - oc.docker:ci:amd64 - allow_failure: true + allow_failure: + exit_codes: 65 timeout: 60 minutes script: - ./scripts/ci/check_commit_messages.sh || exit $? diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 39bfa730658c..ae018a330407 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -454,8 +454,7 @@ let jobs pipeline_type = (* ./scripts/ci/check_commit_messages.sh exits with code 65 when a git history contains invalid commits titles in situations where that is allowed. *) (script_propagate_exit_code "./scripts/ci/check_commit_messages.sh") - (* temporary cf issue https://gitlab.com/tezos/tezos/-/issues/7436 *) - ~allow_failure:Yes + ~allow_failure:(With_exit_codes [65]) in let mr_only_jobs = match pipeline_type with -- GitLab From 893f9583eb60ca4dab051fc61860328fc4459f06 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 3 Sep 2024 17:07:28 +0200 Subject: [PATCH 3/3] CI: adapt [select_tezts.sh] to merged result & merge request pipelines To handle both merged result and merge request pipelines, generalize the solution from [check_commit_messages.sh]. --- scripts/ci/check_commit_messages.sh | 52 +++++------------------ scripts/ci/gitlab_mr_environment.sh | 65 +++++++++++++++++++++++++++++ scripts/ci/select_tezts.sh | 18 ++++---- 3 files changed, 82 insertions(+), 53 deletions(-) create mode 100755 scripts/ci/gitlab_mr_environment.sh diff --git a/scripts/ci/check_commit_messages.sh b/scripts/ci/check_commit_messages.sh index 2ba16766f985..d9df60265b64 100755 --- a/scripts/ci/check_commit_messages.sh +++ b/scripts/ci/check_commit_messages.sh @@ -12,13 +12,9 @@ Checks that the current branch's history, up to MERGE BASE does not contain any commits whose title matches any of the forbidden patterns in ./forbidden_commit_messages.txt -When running locally, MERGE BASE is the best common ancestor between -HEAD and the supposed target branch master (can be set through -CI_MERGE_REQUEST_TARGET_BRANCH_NAME), obtained through -'scripts/ci/git_merge_base.sh'. - -When running in GitLab CI merge request pipeline, the commit history -is checked between MERGE BASE and CI_MERGE_REQUEST_SOURCE_BRANCH_SHA. +MERGE BASE is obtained by passing the variables +TEZOS_CI_MR_{TARGET,HEAD} obtained by +scripts/ci/gitlab_mr_environment.sh to scripts/ci/git_merge_base.sh. This script exits with error code 0 if no forbidden commit subjects are found. If a forbidden commit subject is found in a merge requests @@ -37,38 +33,10 @@ script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" src_dir="$(dirname "$(dirname "$script_dir")")" pattern_file="${script_dir}forbidden_commit_messages.txt" -# Use git_merge_base.sh on merged result pipelines, to ensure we only -# check commit messages of the MR in question. -ORIGIN=${ORIGIN:-origin} - -# CI_MERGE_REQUEST_{SOURCE,TARGET}_BRANCH_SHA are only available in -# merged result pipelines. In merge request pipelines, use -# CI_MERGE_REQUEST_TARGET_BRANCH_NAME instead. -if [ -n "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-}" ] && - [ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_SHA:-}" ]; then - head=${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA} - target=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA} -elif [ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-}" ]; then - head=HEAD - target=${ORIGIN}/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} - - # Make CI_MERGE_REQUEST_TARGET_BRANCH_NAME available locally -- this - # is a prerequisite for 'git_merge_base.sh' when it's operands are - # not fully spelled out hex object names (e.g. when they are branch - # names instead of commit SHAs). - git fetch --depth 1 "${ORIGIN}" "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" -else - car << EOT -None of: - - CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - - CI_MERGE_REQUEST_TARGET_BRANCH_SHA - - CI_MERGE_REQUEST_TARGET_BRANCH_NAME -are set. Cannot retrieve the commit history of this MR. -EOT - exit 1 -fi +# shellcheck source=scripts/ci/gitlab_mr_environment.sh +. "$src_dir"/scripts/ci/gitlab_mr_environment.sh -merge_base=$("$src_dir"/scripts/ci/git_merge_base.sh "$target" "$head" || { +merge_base=$("$src_dir"/scripts/ci/git_merge_base.sh "$TEZOS_CI_MR_TARGET" "$TEZOS_CI_MR_HEAD" || { # Print on stderr to make visible outside command substitution echo "Failed to get merge base, cannot check commit messages." >&2 exit 1 @@ -79,16 +47,16 @@ if [ -z "$merge_base" ]; then fi check_history() { - echo "Check git commit messages in the range ${merge_base}..${head}:" + echo "Check git commit messages in the range ${merge_base}..${TEZOS_CI_MR_HEAD}:" echo "" - git log --format=" - '%s'" "${merge_base}".."${head}" + git log --format=" - '%s'" "${merge_base}".."${TEZOS_CI_MR_HEAD}" echo # Use short flags for grep for busybox-compatibility. - if git log --format="%s" "${merge_base}".."${head}" | grep -qE -f "${pattern_file}" > /dev/null; then + if git log --format="%s" "${merge_base}".."${TEZOS_CI_MR_HEAD}" | grep -qE -f "${pattern_file}" > /dev/null; then # Match commits / forbidden patterns cross wise for user # friendly output. - for commit in $(git log --format="%H" "${merge_base}".."${head}"); do + for commit in $(git log --format="%H" "${merge_base}".."${TEZOS_CI_MR_HEAD}"); do commit_title=$(git show -s --format=%B "${commit}" | head -n1) while read -r pattern; do if echo "${commit_title}" | grep -qE "$pattern"; then diff --git a/scripts/ci/gitlab_mr_environment.sh b/scripts/ci/gitlab_mr_environment.sh new file mode 100755 index 000000000000..c8859b1f4ae8 --- /dev/null +++ b/scripts/ci/gitlab_mr_environment.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +set -eu + +if [ -n "${TRACE:-}" ]; then set -x; fi + +usage() { + cat << EOT +Usage: $0 + +Sets common variables for scripts running in GitLab CI jobs of merge +request pipelines: + +TEZOS_CI_MR_HEAD + + In merge request pipelines, the string 'HEAD'. + In merged results pipelines, the contents of \$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA. + +TEZOS_CI_MR_TARGET + + In merge request pipelines, the string \$ORIGIN/\$CI_MERGE_REQUEST_TARGET_BRANCH_NAME, + where \$ORIGIN defaults to 'origin'. + In merged results pipelines, the contents of '\$CI_MERGE_REQUEST_TARGET_BRANCH_SHA'. + +EOT + exit 1 +} + +if [ "${1:-}" = "--help" ]; then + usage +fi + +ORIGIN=${ORIGIN:-origin} + +export TEZOS_CI_MR_HEAD +export TEZOS_CI_MR_TARGET + +# CI_MERGE_REQUEST_{SOURCE,TARGET}_BRANCH_SHA are only available in +# merged results pipelines. In merge request pipelines, use +# CI_MERGE_REQUEST_TARGET_BRANCH_NAME instead. +if [ -n "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-}" ] && + [ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_SHA:-}" ]; then + # This is a merged results pipeline. + TEZOS_CI_MR_HEAD=${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA} + TEZOS_CI_MR_TARGET=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA} +elif [ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-}" ]; then + # This is a merge request pipeline. + TEZOS_CI_MR_HEAD=HEAD + TEZOS_CI_MR_TARGET=${ORIGIN}/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} + + # Make CI_MERGE_REQUEST_TARGET_BRANCH_NAME available locally -- this + # is a prerequisite for 'git_merge_base.sh' when its operands are + # not fully spelled-out hex object names (e.g. when they are branch + # names instead of commit SHAs). + git fetch --depth 1 "${ORIGIN}" "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" +else + cat << EOT +None of: + - CI_MERGE_REQUEST_SOURCE_BRANCH_SHA + - CI_MERGE_REQUEST_TARGET_BRANCH_SHA + - CI_MERGE_REQUEST_TARGET_BRANCH_NAME +are set. Cannot set TEZOS_CI_MR_HEAD and TEZOS_CI_MR_TARGET. +EOT + exit 1 +fi diff --git a/scripts/ci/select_tezts.sh b/scripts/ci/select_tezts.sh index 057a5bb44237..cfcc8376f140 100755 --- a/scripts/ci/select_tezts.sh +++ b/scripts/ci/select_tezts.sh @@ -18,14 +18,10 @@ abort() { exit 17 } -# If set, diff base against CI_MERGE_REQUEST_SOURCE_BRANCH_SHA instead of HEAD. -# This variable contains the HEAD SHA of the merge request's source branch. -# On merged result pipelines, this is not equivalent on the local git's HEAD, -# as the merged result pipeline runs on an ephemeral, rebased branch. -head=${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-HEAD} -target=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA:-${ORIGIN}/master} - -merge_base=$("$src_dir"/scripts/ci/git_merge_base.sh "$target" "$head" || { +# shellcheck source=scripts/ci/gitlab_mr_environment.sh +. "$src_dir"/scripts/ci/gitlab_mr_environment.sh + +merge_base=$("$src_dir"/scripts/ci/git_merge_base.sh "$TEZOS_CI_MR_TARGET" "$TEZOS_CI_MR_HEAD" || { # Print on stderr to make visible outside command substitution echo "Failed to get merge base, test selection is disabled." >&2 abort @@ -35,8 +31,8 @@ if [ -z "$merge_base" ]; then abort fi -echo "---- Fetching source branch HEAD ($head) and base ($merge_base) from $ORIGIN..." -if ! git fetch "$ORIGIN" "$head" "$merge_base"; then +echo "---- Fetching source branch HEAD ($TEZOS_CI_MR_HEAD) and base ($merge_base) from $ORIGIN..." +if ! git fetch "$ORIGIN" "$TEZOS_CI_MR_HEAD" "$merge_base"; then # Example error that was seen in a job: # error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: INTERNAL_ERROR (err 2) # error: 50483 bytes of body are still expected @@ -47,7 +43,7 @@ if ! git fetch "$ORIGIN" "$head" "$merge_base"; then fi echo "---- Diffing..." -CHANGES="$(git diff --name-only "$merge_base" "$head")" +CHANGES="$(git diff --name-only "$merge_base" "$TEZOS_CI_MR_HEAD")" echo "$CHANGES" echo "---- Compiling manifest/manifest..." -- GitLab