Skip to content

Commit 38a33e5

Browse files
joao-paulo-parityjoao-paulo-parity
committed
consider all companions from all PRs descriptions
1 parent 9a1b2d0 commit 38a33e5

File tree

1 file changed

+127
-107
lines changed

1 file changed

+127
-107
lines changed

‎check_dependent_project.sh

Lines changed: 127 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,13 @@ github_api_token="$5"
3939
update_crates_on_default_branch="$6"
4040

4141
this_repo_dir="$PWD"
42+
companions_dir="$this_repo_dir/companions"
4243
github_api="https://api.github.com"
4344

45+
# valid for 69ab0f76fb851968af8e493061cca84a2f3b1c5b
46+
# FIXME: extract this information from the diener CLI when that is supported
47+
diener_patch_targets=(substrate polkadot cumulus)
48+
4449
our_crates=()
4550
our_crates_source="git+https://github.com/$org/$this_repo"
4651
discover_our_crates() {
@@ -150,73 +155,142 @@ match_their_crates() {
150155
fi
151156
}
152157

153-
patch_and_check_dependent() {
154-
match_their_crates "$(basename "$PWD")"
155-
156-
# Update the crates to the latest version.
157-
#
158-
# This is for example needed if there was a pr to Substrate that only required a Polkadot companion
159-
# and Cumulus wasn't yet updated to use the latest commit of Polkadot.
160-
for update in $update_crates_on_default_branch; do
161-
cargo update -p $update
162-
done
163-
164-
diener patch --crates-to-patch "$this_repo_dir" "$this_repo_diener_arg" --path "Cargo.toml"
165-
eval "${COMPANION_CHECK_COMMAND:-cargo check --all-targets --workspace}"
166-
}
158+
companions=()
159+
process_pr_description_line() {
160+
local companion_expr="$1"
167161

168-
process_companion_pr() {
169162
# e.g. https://github.com/paritytech/polkadot/pull/123
170163
# or polkadot#123
171-
local companion_expr="$1"
172164
if
173165
[[ "$companion_expr" =~ ^https://github\.com/$org/([^/]+)/pull/([[:digit:]]+) ]] ||
174166
[[ "$companion_expr" =~ ^$org/([^#]+)#([[:digit:]]+) ]] ||
175-
[[ "$companion_expr" =~ ^([^#]+)#([[:digit:]]+) ]]; then
176-
local companion_repo="${BASH_REMATCH[1]}"
177-
local companion_pr_number="${BASH_REMATCH[2]}"
178-
echo "Parsed companion_repo=$companion_repo and companion_pr_number=$companion_pr_number from $companion_expr (trying to match companion_repo=$dependent_repo)"
167+
[[ "$companion_expr" =~ ^([^#]+)#([[:digit:]]+) ]]
168+
then
169+
local repo="${BASH_REMATCH[1]}"
170+
local pr_number="${BASH_REMATCH[2]}"
171+
172+
echo "Parsed companion repo=$repo and pr_number=$pr_number from $companion_expr"
173+
174+
if [ "$this_repo" == "$repo" ]; then
175+
echo "Skipping $companion_expr it refers to the repository where this script is currently running"
176+
return
177+
fi
178+
179+
# keep track of duplicated companion references not only to avoid useless
180+
# work but also to avoid infinite mutual recursion when 2+ PRs reference
181+
# each other
182+
for comp in "${companions[@]}"; do
183+
if [ "$comp" == "$repo" ]; then
184+
echo "Skipping $companion_expr as the repository $repo has already been registered before"
185+
return
186+
fi
187+
done
188+
companions+=("$repo")
189+
190+
git clone --depth=1 "https://github.com/$org/$repo.git" "$companions_dir/$repo"
191+
pushd "$repo" >/dev/null
192+
local ref="$(curl \
193+
-sSL \
194+
-H "Authorization: token $github_api_token" \
195+
"$github_api/repos/$org/$repo/pulls/$pr_number" | \
196+
jq -e -r ".head.ref // error(\"$repo#$pr_number is missing head.ref\"))"
197+
)"
198+
git fetch --depth=1 origin "pull/$pr_number/head:$ref"
199+
git checkout "$ref"
200+
popd >/dev/null
201+
202+
# collect also the companions of companions
203+
process_pr_description "$repo" "$pr_number"
179204
else
180205
die "Companion PR description had invalid format or did not belong to organization $org: $companion_expr"
181206
fi
207+
}
182208

183-
if [ "$companion_repo" != "$dependent_repo" ]; then
184-
echo "companion repo $companion_repo doesn't match dependent repo $dependent_repo. Check that you pasted the pure link to the companion."
209+
process_pr_description() {
210+
local repo="$1"
211+
local pr_number="$2"
212+
213+
if ! [[ "$pr_number" =~ ^[[:digit:]]+$ ]]; then
185214
return
186215
fi
187216

188-
was_companion_found=true
217+
echo "processing pull request $repo#$pr_number"
189218

190-
read -d '\n' -r mergeable pr_head_ref pr_head_sha < <(curl \
219+
local lines=()
220+
while IFS= read -r line; do
221+
lines+=("$line")
222+
done < <(curl \
191223
-sSL \
192224
-H "Authorization: token $github_api_token" \
193-
"$github_api/repos/$org/$companion_repo/pulls/$companion_pr_number" | \
194-
jq -r "[
195-
.mergeable // error(\"Companion $companion_expr is not mergeable\"),
196-
.head.ref // error(\"Missing .head.ref from API data of $companion_expr\"),
197-
.head.sha // error(\"Missing .head.sha from API data of $companion_expr\")
198-
] | .[]"
199-
# https://stackoverflow.com/questions/40547032/bash-read-returns-with-exit-code-1-even-though-it-runs-as-expected
200-
# ignore the faulty exit code since read still is regardless still reading the values we want
201-
) || :
202-
203-
local expected_mergeable=true
204-
if [ "$mergeable" != "$expected_mergeable" ]; then
205-
die "Github API says $companion_expr is not mergeable (got $mergeable, expected $expected_mergeable)"
225+
"$github_api/repos/$org/$this_repo/pulls/$CI_COMMIT_REF_NAME" | \
226+
jq -e -r ".body"
227+
)
228+
if [ ! "${lines[@]:-}" ]; then
229+
die "No lines were read for the description of PR $pr_number (some error probably occurred)"
206230
fi
207231

208-
echo
209-
echo "merging master into the pr..."
210-
git clone --depth 100 "https://github.com/$org/$companion_repo.git"
211-
pushd "$companion_repo" >/dev/null
212-
git fetch origin "pull/$companion_pr_number/head:$pr_head_ref"
213-
git checkout "$pr_head_sha"
214-
git merge master --verbose --no-edit -m "master was merged into the pr by check_dependent_project.sh process_companion_pr()"
215-
echo "done"
216-
echo
232+
# first check if the companion is disabled *somewhere* in the PR description
233+
# before doing any work
234+
for line in "${lines[@]}"; do
235+
if
236+
[[ "$line" =~ skip[^[:alnum:]]+([^[:space:]]+) ]] &&
237+
[[ "$repo" == "$dependent_repo" ]] &&
238+
[[
239+
"${BASH_REMATCH[1]}" = "$CI_JOB_NAME" ||
240+
"${BASH_REMATCH[1]}" = "continuous-integration/gitlab-$CI_JOB_NAME"
241+
]]
242+
then
243+
# FIXME: This escape hatch should be removed at some point when the
244+
# companion build system is able to deal with all edge cases, such as
245+
# the one described in
246+
# https://github.com/paritytech/pipeline-scripts/issues/3#issuecomment-947539791
247+
echo "Skipping $CI_JOB_NAME as specified in the PR description"
248+
exit
249+
fi
250+
done
251+
252+
for line in "${lines[@]}"; do
253+
if [[ "$line" =~ [cC]ompanion:[[:space:]]*([^[:space:]]+) ]]; then
254+
echo "Detected companion in the PR description of $repo#$pr_number: ${BASH_REMATCH[1]}"
255+
process_pr_description_line "${BASH_REMATCH[1]}"
256+
fi
257+
done
258+
}
259+
260+
patch_and_check_dependent() {
261+
local dependent="$1"
262+
263+
pushd "$dependent" >/dev/null
217264

218-
echo "running checks for the companion $companion_expr of $companion_repo"
219-
patch_and_check_dependent
265+
match_their_crates "$dependent"
266+
267+
# Update the crates to the latest version.
268+
#
269+
# This is for example needed if there was a pr to Substrate that only required a Polkadot companion
270+
# and Cumulus wasn't yet updated to use the latest commit of Polkadot.
271+
for update in $update_crates_on_default_branch; do
272+
cargo update -p "$update"
273+
done
274+
275+
for comp in "${companions[@]}"; do
276+
local found
277+
for diener_target in "${diener_patch_targets[@]}"; do
278+
if [ "$diener_target" = "$comp" ]; then
279+
echo "Patching $comp into $dependent"
280+
diener patch --crates-to-patch "--$diener_target" "$companions_dir/$comp" --path "Cargo.toml"
281+
found=true
282+
break
283+
fi
284+
done
285+
if [ "${found:-}" ]; then
286+
unset found
287+
else
288+
echo "NOTE: Companion $comp was specified but not patched through diener. Perhaps diener does not support it."
289+
fi
290+
done
291+
292+
diener patch --crates-to-patch "$this_repo_dir" "$this_repo_diener_arg" --path "Cargo.toml"
293+
eval "${COMPANION_CHECK_COMMAND:-cargo check --all-targets --workspace}"
220294

221295
popd >/dev/null
222296
}
@@ -234,70 +308,16 @@ main() {
234308
# Note that the target branch might not actually be master, but we default to it in the assumption
235309
# of the common case. This could be refined in the future.
236310
git fetch origin +master:master
237-
git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME
238-
git checkout $CI_COMMIT_REF_NAME
311+
git fetch origin "+$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME"
312+
git checkout "$CI_COMMIT_REF_NAME"
239313
git merge master --verbose --no-edit -m "master was merged into the pr by check_dependent_project.sh main()"
240-
echo "done"
314+
echo "merging master into the pr: done"
241315
echo
242316

243317
discover_our_crates
244318

245-
if [[ "$CI_COMMIT_REF_NAME" =~ ^[[:digit:]]+$ ]]; then
246-
echo "this is pull request number $CI_COMMIT_REF_NAME"
247-
248-
# workaround for early exits not being detected in command substitution
249-
# https://unix.stackexchange.com/questions/541969/nested-command-substitution-does-not-stop-a-script-on-a-failure-even-if-e-and-s
250-
local last_line
251-
252-
while IFS= read -r line; do
253-
last_line="$line"
254-
255-
if [[ "$line" =~ [cC]ompanion:[[:space:]]*([^[:space:]]+) ]]; then
256-
echo "Detected companion in PR description: ${BASH_REMATCH[1]}"
257-
process_companion_pr "${BASH_REMATCH[1]}"
258-
elif [[ "$line" =~ skip[^[:alnum:]]+([^[:space:]]+) ]]; then
259-
# FIXME: This escape hatch should be removed at some point when the
260-
# companion build system is able to deal with all edge cases, such as
261-
# the one described in
262-
# https://github.com/paritytech/pipeline-scripts/issues/3#issuecomment-947539791
263-
if [[
264-
"${BASH_REMATCH[1]}" = "$CI_JOB_NAME" ||
265-
"${BASH_REMATCH[1]}" = "continuous-integration/gitlab-$CI_JOB_NAME"
266-
]]; then
267-
echo "Skipping $CI_JOB_NAME as specified in the PR description"
268-
exit
269-
fi
270-
fi
271-
done < <(curl \
272-
-sSL \
273-
-H "Authorization: token $GITHUB_TOKEN" \
274-
"$github_api/repos/$org/$this_repo/pulls/$CI_COMMIT_REF_NAME" | \
275-
jq -e -r ".body"
276-
)
277-
if [ -z "${last_line+_}" ]; then
278-
die "No lines were read for the description of PR $companion_pr_number (some error probably occurred)"
279-
fi
280-
fi
281-
282-
if [ "${was_companion_found:-}" ]; then
283-
exit
284-
fi
285-
286-
echo "running checks for the default branch of $dependent_repo"
287-
288-
git clone --depth 1 "https://github.com/$org/$dependent_repo.git"
289-
pushd "$dependent_repo" >/dev/null
319+
process_pr_description "$this_repo" "$CI_COMMIT_REF_NAME"
290320

291-
# Update the crates to the latest version.
292-
#
293-
# This is for example needed if there was a pr to Substrate that only required a Polkadot companion
294-
# and Cumulus wasn't yet updated to use the latest commit of Polkadot.
295-
for update in $update_crates_on_default_branch; do
296-
cargo update -p $update
297-
done
298-
299-
patch_and_check_dependent
300-
301-
popd >/dev/null
321+
patch_and_check_dependent "$dependent_repo"
302322
}
303323
main

0 commit comments

Comments
 (0)