diff options
Diffstat (limited to 'afdo_tools/update_kernel_afdo')
| -rwxr-xr-x | afdo_tools/update_kernel_afdo | 303 |
1 files changed, 217 insertions, 86 deletions
diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo index a677bf1b..ff0ab224 100755 --- a/afdo_tools/update_kernel_afdo +++ b/afdo_tools/update_kernel_afdo @@ -5,114 +5,245 @@ # Due to crbug.com/1081332, we need to update AFDO metadata # manually. This script performs a few checks and generates a -# new kernel_afdo.json file, which can then be committed. -# -# USAGE: -# toolchain-utils$ ./afdo_tools/update_kernel_afdo -# -# The script modifies the JSON file and shows the git diff. -# -# If the changes look good, git commit them. Example commit -# message (from crrev.com/c/2197462): -# -# afdo_metadata: Publish the new kernel profiles -# -# Update chromeos-kernel-3_18 to R84-13080.0-1589189810 -# Update chromeos-kernel-4_4 to R84-13080.0-1589189726 -# Update chromeos-kernel-4_14 to R84-13080.0-1589190025 -# Update chromeos-kernel-4_19 to R84-13080.0-1589189550 -# -# BUG=None -# TEST=Verified in kernel-release-afdo-verify-orchestrator. +# new kernel_afdo.json file, which can then be submitted. # +USAGE=" +Usage: $(basename $0) [main|beta|stable|all] [--help] + +Description: + The script takes one optional argument which is the channel where we want +to update the kernel afdo and creates a commit (or commits with \"all\" +channels) in the corresponding branch. + No arguments defaults to \"all\". + Follow the prompt to submit the changes. + NO CLEAN-UP NEEDED. The script ignores any local changes and keeps +the current branch unchanged. +" + set -eu set -o pipefail -CROS_REPO=https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel -KVERS="3.18 4.4 4.14 4.19" -errs="" -successes=0 +KVERS="4.4 4.14 4.19 5.4" +failed_channels="" +# Add skipped chrome branches in ascending order here. +SKIPPED_BRANCHES="95" script_dir=$(dirname "$0") -tc_utils_dir="$script_dir/.." -metadata_dir="$tc_utils_dir/afdo_metadata" -outfile="$metadata_dir/kernel_afdo.json" +tc_utils_dir="${script_dir}/.." +metadata_dir="${tc_utils_dir}/afdo_metadata" +outfile="$(realpath --relative-to="${tc_utils_dir}" \ + "${metadata_dir}"/kernel_afdo.json)" +# Convert toolchain_utils into the absolute path. +abs_tc_utils_dir="$(realpath ${tc_utils_dir})" -# The most recent Monday, in Unix timestamp format. -if [ $(date +%a) = "Mon" ] -then - expected_time=$(date +%s -d 00:00:00) -else - expected_time=$(date +%s -d "last Monday") -fi +# Check profiles uploaded within the last week. +expected_time=$(date +%s -d "week ago") + +declare -A branch branch_number commit +remote_repo=$(git -C "${tc_utils_dir}" remote) +canary_ref="refs/heads/main" +# Read the last two release-Rxx from remote branches +# and assign them to stable_ref and beta_ref. +# sort -V is the version sort which puts R100 after R99. +last_branches=$(git -C "${tc_utils_dir}" ls-remote -h "${remote_repo}" \ + release-R\* | cut -f2 | sort -V | tail -n 2) +# We need `echo` to convert newlines into spaces for read. +read stable_ref beta_ref <<< $(echo ${last_branches}) +# Branch names which start from release-R. +branch["beta"]=${beta_ref##*/} +branch["stable"]=${stable_ref##*/} +branch["canary"]=${canary_ref##*/} -# Get the current canary branch number (using beta + 1) -beta=$(git ls-remote -h $CROS_REPO | \ - sed -n -e "s/^.*release-R\([0-9][0-9]*\).*$/\1/p" | \ - sort -g | tail -1) -canary="$(($beta + 1))" +# Get current branch numbers (number which goes after R). +branch_number["stable"]=$(echo "${branch["stable"]}" | \ + sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") +branch_number["beta"]=$(echo "${branch["beta"]}" | \ + sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") +branch_number["canary"]="$((branch_number[beta] + 1))" +for skipped_branch in $SKIPPED_BRANCHES ; do + if [[ ${branch_number["canary"]} == $skipped_branch ]] ; then + ((branch_number[canary]++)) + fi +done + +# Without arguments the script updates all branches. +channels=${1:-"all"} +case "${channels}" in + stable | canary | beta ) + ;; + main ) + channels="canary" + ;; + all ) + channels="canary beta stable" + ;; + --help | help | -h ) + echo "$USAGE" + exit 0 + ;; + * ) + echo "Channel \"${channels}\" is not supported. +Must be main (or canary), beta, stable or all." >&2 + echo "$USAGE" + exit 1 +esac -json="{" -sep="" -for kver in $KVERS +# Fetch latest branches. +git -C "${tc_utils_dir}" fetch "${remote_repo}" + +worktree_dir=$(mktemp -d) +echo "-> Working in ${worktree_dir}" +# Create a worktree and make changes there. +# This way we don't need to clean-up and sync toolchain_utils before the +# change. Neither we should care about clean-up after the submit. +git -C "${tc_utils_dir}" worktree add --detach "${worktree_dir}" +trap "git -C ${abs_tc_utils_dir} worktree remove ${worktree_dir}" EXIT +cd "${worktree_dir}" + +for channel in ${channels} do - # Sort the gs output by timestamp (default ordering is by name, so - # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz) - latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ - grep "R${canary}" | tail -1 || true) - if [ -z "$latest" ] - then - # if no profiles exist for R${canary}, try the previous branch + errs="" + successes=0 + curr_branch_number=${branch_number[${channel}]} + curr_branch=${branch[${channel}]} + echo + echo "Checking \"${channel}\" channel..." + echo "branch_number=${curr_branch_number} branch=${curr_branch}" + json="{" + sep="" + for kver in $KVERS + do + # Sort the gs output by timestamp (default ordering is by name, so + # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz) latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ - grep "R${beta}" | tail -1) - fi + grep "R${curr_branch_number}" | tail -1 || true) + if [[ -z "$latest" && "${channel}" != "stable" ]] + then + # if no profiles exist for the current branch, try the previous branch + latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ + grep "R$((curr_branch_number - 1))" | tail -1) + fi - # Verify that the file has the expected date. - file_time=$(echo "$latest" | awk '{print $2}') - file_time_unix=$(date +%s -d "$file_time") - if [ $file_time_unix -lt $expected_time ] - then - expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time) - echo "Wrong date for $kver: $file_time is before $expected" >&2 - errs="$errs $kver" - continue - fi + # Verify that the file has the expected date. + file_time=$(echo "$latest" | awk '{print $2}') + file_time_unix=$(date +%s -d "$file_time") + if [ $file_time_unix -lt $expected_time ] + then + expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time) + echo "Wrong date for $kver: $file_time is before $expected" >&2 + errs="$errs $kver" + continue + fi - # Generate JSON. - json_kver=$(echo "$kver" | tr . _) - # b/147370213 (migrating profiles from gcov format) may result in the - # pattern below no longer doing the right thing. - name=$(echo "$latest" | sed 's%.*/\(.*\)\.gcov.*%\1%') - json=$(cat <<EOT + # Generate JSON. + json_kver=$(echo "$kver" | tr . _) + # b/147370213 (migrating profiles from gcov format) may result in the + # pattern below no longer doing the right thing. + name=$(echo "$latest" | sed 's%.*/\(.*\)\.gcov.*%\1%') + json=$(cat <<EOT $json$sep "chromeos-kernel-$json_kver": { "name": "$name" } EOT -) - sep="," - successes=$((successes + 1)) -done + ) + sep="," + successes=$((successes + 1)) + done -# If we did not succeed for any kvers, exit now. -if [ $successes -eq 0 ] -then - echo "error: AFDO profiles out of date for all kernel versions" >&2 - exit 2 -fi + # If we did not succeed for any kvers, exit now. + if [[ $successes -eq 0 ]] + then + echo "error: AFDO profiles out of date for all kernel versions" >&2 + failed_channels="${failed_channels} ${channel}" + continue + fi + + git reset --hard HEAD + echo git checkout "${remote_repo}/${curr_branch}" + git checkout "${remote_repo}/${curr_branch}" + + # Write new JSON file. + # Don't use `echo` since `json` might have esc characters in it. + printf "%s\n}\n" "$json" > "$outfile" + + # If no changes were made, say so. + outdir=$(dirname "$outfile") + shortstat=$(cd "$outdir" && git status --short $(basename "$outfile")) + [ -z "$shortstat" ] && echo $(basename "$outfile")" is up to date." \ + && continue + + # If we had any errors, warn about them. + if [[ -n "$errs" ]] + then + echo "warning: failed to update $errs in ${channel}" >&2 + failed_channels="${failed_channels} ${channel}" + continue + fi + + git add afdo_metadata/kernel_afdo.json + case "${channel}" in + canary ) + commit_contents="afdo_metadata: Publish the new kernel profiles -# Write new JSON file. -printf "%s\n}\n" "$json" > "$outfile" +Update chromeos-kernel-4_4 +Update chromeos-kernel-4_14 +Update chromeos-kernel-4_19 +Update chromeos-kernel-5_4 -# Show the changes. -(cd "$tc_utils_dir" && git diff) +BUG=None +TEST=Verified in kernel-release-afdo-verify-orchestrator" + ;; + beta | stable ) + commit_contents="afdo_metadata: Publish the new kernel profiles\ + in ${curr_branch} -# If no changes were made, say so. -outdir=$(dirname "$outfile") -shortstat=$(cd "$outdir" && git status --short $(basename "$outfile")) -[ -n "$shortstat" ] || echo $(basename "$outfile")" is up to date." +Have PM pre-approval because this shouldn't break the release branch. -# If we had any errors, warn about them. -[ -z "$errs" ] || echo "warning: failed to update$errs" >&2 +BUG=None +TEST=Verified in kernel-release-afdo-verify-orchestrator" + ;; + * ) + echo "internal error: unhandled channel \"${channel}\"" >&2 + exit 2 + esac + + git commit -v -e -m "${commit_contents}" + + commit[${channel}]=$(git -C "${worktree_dir}" rev-parse HEAD) +done + +echo +# Array size check doesn't play well with the unbound variable option. +set +u +if [[ ${#commit[@]} -gt 0 ]] +then + set -u + echo "The change is applied in ${!commit[@]}." + echo "Run these commands to submit the change:" + echo + for channel in ${!commit[@]} + do + echo -e "\tgit -C ${tc_utils_dir} push ${remote_repo} \ +${commit[${channel}]}:refs/for/${branch[${channel}]}" + done + + # Report failed channels. + if [[ -n "${failed_channels}" ]] + then + echo + echo "error: failed to update kernel afdo in ${failed_channels}" >&2 + exit 3 + fi +else + # No commits. Check if it is due to failures. + if [[ -z "${failed_channels}" ]] + then + echo "No changes are applied. It looks like AFDO versions are up to date." + else + echo "error: update in ${failed_channels} failed" >&2 + exit 3 + fi +fi |