From 859aa637a00c105ffe4570b324fe32e0fd3716a9 Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Fri, 19 Mar 2021 10:39:47 -0700 Subject: afdo: Add 5.4 into kernel afdo metadata 5.4 kernel afdo generate builder was recently added. Add 5.4 into the list for pupr updates. BUG=b:180725525 TEST=`./afdo_tools/update_kernel_afdo` creates 5.4 data. Change-Id: If43727dbaf0fd6409f043d5978795219b580fcb5 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2774636 Tested-by: Denis Nikitin Reviewed-by: Caroline Tice --- afdo_tools/update_kernel_afdo | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'afdo_tools/update_kernel_afdo') diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo index a677bf1b..2c6ba266 100755 --- a/afdo_tools/update_kernel_afdo +++ b/afdo_tools/update_kernel_afdo @@ -21,6 +21,7 @@ # Update chromeos-kernel-4_4 to R84-13080.0-1589189726 # Update chromeos-kernel-4_14 to R84-13080.0-1589190025 # Update chromeos-kernel-4_19 to R84-13080.0-1589189550 +# Update chromeos-kernel-5_4 to R84-13080.0-1589189550 # # BUG=None # TEST=Verified in kernel-release-afdo-verify-orchestrator. @@ -31,7 +32,7 @@ set -o pipefail CROS_REPO=https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel -KVERS="3.18 4.4 4.14 4.19" +KVERS="3.18 4.4 4.14 4.19 5.4" errs="" successes=0 -- cgit v1.2.3 From 3ae442199e9665d9a13c9f9f4b4306acefcc9b3c Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Sun, 2 May 2021 23:45:03 -0700 Subject: afdo_tools: Improve update_kernel_afdo script By default the script now will create a commit and give a prompt on how to submit the change. Neither sync or clean-up needed to submit the change. The new workflow doesn't touch existing checkout and ignores (and keeps) any local changes. Without arguments the script tries to update metadata in all channels: canary (main), beta and stable. To update afdo metadata in specific channel run: ./update_kernel_afdo canary|beta|stable. BUG=None TEST=afdo_tools/update_kernel_afdo produces 3 commits in release-R92, release-R91 and release-R90 Change-Id: I61c8c743a4634d2ab4e4837f8a31a31a6cea2c2a Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2872860 Tested-by: Denis Nikitin Reviewed-by: Caroline Tice Reviewed-by: George Burgess Commit-Queue: Denis Nikitin --- afdo_tools/update_kernel_afdo | 286 ++++++++++++++++++++++++++++++------------ 1 file changed, 207 insertions(+), 79 deletions(-) (limited to 'afdo_tools/update_kernel_afdo') diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo index 2c6ba266..aa14b44a 100755 --- a/afdo_tools/update_kernel_afdo +++ b/afdo_tools/update_kernel_afdo @@ -5,41 +5,36 @@ # Due to crbug.com/1081332, we need to update AFDO metadata # manually. This script performs a few checks and generates a -# new kernel_afdo.json file, which can then be committed. -# -# USAGE: -# toolchain-utils$ ./afdo_tools/update_kernel_afdo -# -# The script modifies the JSON file and shows the git diff. -# -# If the changes look good, git commit them. Example commit -# message (from crrev.com/c/2197462): -# -# afdo_metadata: Publish the new kernel profiles -# -# Update chromeos-kernel-3_18 to R84-13080.0-1589189810 -# Update chromeos-kernel-4_4 to R84-13080.0-1589189726 -# Update chromeos-kernel-4_14 to R84-13080.0-1589190025 -# Update chromeos-kernel-4_19 to R84-13080.0-1589189550 -# Update chromeos-kernel-5_4 to R84-13080.0-1589189550 -# -# BUG=None -# TEST=Verified in kernel-release-afdo-verify-orchestrator. +# new kernel_afdo.json file, which can then be submitted. # +USAGE=" +Usage: $(basename $0) [main|beta|stable|all] [--help] + +Description: + The script takes one optional argument which is the channel where we want +to update the kernel afdo and creates a commit (or commits with \"all\" +channels) in the corresponding branch. + No arguments defaults to \"all\". + Follow the prompt to submit the changes. + NO CLEAN-UP NEEDED. The script ignores any local changes and keeps +the current branch unchanged. +" + set -eu set -o pipefail -CROS_REPO=https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel KVERS="3.18 4.4 4.14 4.19 5.4" -errs="" -successes=0 +failed_channels="" script_dir=$(dirname "$0") -tc_utils_dir="$script_dir/.." -metadata_dir="$tc_utils_dir/afdo_metadata" -outfile="$metadata_dir/kernel_afdo.json" +tc_utils_dir="${script_dir}/.." +metadata_dir="${tc_utils_dir}/afdo_metadata" +outfile="$(realpath --relative-to="${tc_utils_dir}" \ + "${metadata_dir}"/kernel_afdo.json)" +# Convert toolchain_utils into the absolute path. +abs_tc_utils_dir="$(realpath ${tc_utils_dir})" # The most recent Monday, in Unix timestamp format. if [ $(date +%a) = "Mon" ] @@ -49,71 +44,204 @@ else expected_time=$(date +%s -d "last Monday") fi -# Get the current canary branch number (using beta + 1) -beta=$(git ls-remote -h $CROS_REPO | \ - sed -n -e "s/^.*release-R\([0-9][0-9]*\).*$/\1/p" | \ - sort -g | tail -1) -canary="$(($beta + 1))" +declare -A branch branch_number commit +remote_repo=$(git -C "${tc_utils_dir}" remote) +canary_ref="refs/heads/main" +# Read the last two release-Rxx from remote branches +# and assign them to stable_ref and beta_ref. +# sort -V is the version sort which puts R100 after R99. +last_branches=$(git -C "${tc_utils_dir}" ls-remote -h "${remote_repo}" \ + release-R\* | cut -f2 | sort -V | tail -n 2) +# We need `echo` to convert newlines into spaces for read. +read stable_ref beta_ref <<< $(echo ${last_branches}) +# Branch names which start from release-R. +branch["beta"]=${beta_ref##*/} +branch["stable"]=${stable_ref##*/} +branch["canary"]=${canary_ref##*/} + +# Get current branch numbers (number which goes after R). +branch_number["stable"]=$(echo "${branch["stable"]}" | \ + sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") +branch_number["beta"]="$((branch_number[stable] + 1))" +branch_number["canary"]="$((branch_number[beta] + 1))" + +# Without arguments the script updates all branches. +channels=${1:-"all"} +case "${channels}" in + stable | canary | beta ) + ;; + main ) + channels="canary" + ;; + all ) + channels="canary beta stable" + ;; + --help | help | -h ) + echo "$USAGE" + exit 0 + ;; + * ) + echo "Channel \"${channels}\" is not supported. +Must be main (or canary), beta, stable or all." >&2 + echo "$USAGE" + exit 1 +esac + +# Fetch latest branches. +git -C "${tc_utils_dir}" fetch "${remote_repo}" -json="{" -sep="" -for kver in $KVERS +worktree_dir=$(mktemp -d) +echo "-> Working in ${worktree_dir}" +# Create a worktree and make changes there. +# This way we don't need to clean-up and sync toolchain_utils before the +# change. Neither we should care about clean-up after the submit. +git -C "${tc_utils_dir}" worktree add --detach "${worktree_dir}" +trap "git -C ${abs_tc_utils_dir} worktree remove ${worktree_dir}" EXIT +cd "${worktree_dir}" + +for channel in ${channels} do - # Sort the gs output by timestamp (default ordering is by name, so - # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz) - latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ - grep "R${canary}" | tail -1 || true) - if [ -z "$latest" ] - then - # if no profiles exist for R${canary}, try the previous branch + errs="" + successes=0 + curr_branch_number=${branch_number[${channel}]} + curr_branch=${branch[${channel}]} + echo + echo "Checking \"${channel}\" channel..." + echo "branch_number=${curr_branch_number} branch=${curr_branch}" + json="{" + sep="" + for kver in $KVERS + do + # Sort the gs output by timestamp (default ordering is by name, so + # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz) latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ - grep "R${beta}" | tail -1) - fi + grep "R${curr_branch_number}" | tail -1 || true) + if [[ -z "$latest" && "${channel}" != "stable" ]] + then + # if no profiles exist for the current branch, try the previous branch + latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ + grep "R$((curr_branch_number - 1))" | tail -1) + fi - # Verify that the file has the expected date. - file_time=$(echo "$latest" | awk '{print $2}') - file_time_unix=$(date +%s -d "$file_time") - if [ $file_time_unix -lt $expected_time ] - then - expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time) - echo "Wrong date for $kver: $file_time is before $expected" >&2 - errs="$errs $kver" - continue - fi + # Verify that the file has the expected date. + file_time=$(echo "$latest" | awk '{print $2}') + file_time_unix=$(date +%s -d "$file_time") + if [ $file_time_unix -lt $expected_time ] + then + expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time) + echo "Wrong date for $kver: $file_time is before $expected" >&2 + errs="$errs $kver" + continue + fi - # Generate JSON. - json_kver=$(echo "$kver" | tr . _) - # b/147370213 (migrating profiles from gcov format) may result in the - # pattern below no longer doing the right thing. - name=$(echo "$latest" | sed 's%.*/\(.*\)\.gcov.*%\1%') - json=$(cat <&2 - exit 2 -fi + # If we did not succeed for any kvers, exit now. + if [[ $successes -eq 0 ]] + then + echo "error: AFDO profiles out of date for all kernel versions" >&2 + failed_channels="${failed_channels} ${channel}" + continue + fi + + git reset --hard HEAD + echo git checkout "${remote_repo}/${curr_branch}" + git checkout "${remote_repo}/${curr_branch}" + + # Write new JSON file. + # Don't use `echo` since `json` might have esc characters in it. + printf "%s\n}\n" "$json" > "$outfile" + + # If no changes were made, say so. + outdir=$(dirname "$outfile") + shortstat=$(cd "$outdir" && git status --short $(basename "$outfile")) + [ -z "$shortstat" ] && echo $(basename "$outfile")" is up to date." \ + && continue + + # If we had any errors, warn about them. + if [[ -n "$errs" ]] + then + echo "warning: failed to update $errs in ${channel}" >&2 + failed_channels="${failed_channels} ${channel}" + continue + fi + + git add afdo_metadata/kernel_afdo.json + case "${channel}" in + canary ) + commit_contents="afdo_metadata: Publish the new kernel profiles + +Update chromeos-kernel-3_18 +Update chromeos-kernel-4_4 +Update chromeos-kernel-4_14 +Update chromeos-kernel-4_19 +Update chromeos-kernel-5_4 + +BUG=None +TEST=Verified in kernel-release-afdo-verify-orchestrator" + ;; + beta | stable ) + commit_contents="afdo_metadata: Publish the new kernel profiles\ + in ${curr_branch} -# Write new JSON file. -printf "%s\n}\n" "$json" > "$outfile" +Have PM pre-approval because this shouldn't break the release branch. -# Show the changes. -(cd "$tc_utils_dir" && git diff) +BUG=None +TEST=Verified in kernel-release-afdo-verify-orchestrator" + ;; + * ) + echo "internal error: unhandled channel \"${channel}\"" >&2 + exit 2 + esac -# If no changes were made, say so. -outdir=$(dirname "$outfile") -shortstat=$(cd "$outdir" && git status --short $(basename "$outfile")) -[ -n "$shortstat" ] || echo $(basename "$outfile")" is up to date." + git commit -v -e -m "${commit_contents}" -# If we had any errors, warn about them. -[ -z "$errs" ] || echo "warning: failed to update$errs" >&2 + commit[${channel}]=$(git -C "${worktree_dir}" rev-parse HEAD) +done + +echo +# Array size check doesn't play well with the unbound variable option. +set +u +if [[ ${#commit[@]} -gt 0 ]] +then + set -u + echo "The change is applied in ${!commit[@]}." + echo "Run these commands to submit the change:" + echo + for channel in ${!commit[@]} + do + echo -e "\tgit -C ${tc_utils_dir} push ${remote_repo} \ +${commit[${channel}]}:refs/for/${branch[${channel}]}" + done + + # Report failed channels. + if [[ -n "${failed_channels}" ]] + then + echo + echo "error: failed to update kernel afdo in ${failed_channels}" >&2 + exit 3 + fi +else + # No commits. Check if it is due to failures. + if [[ -z "${failed_channels}" ]] + then + echo "No changes are applied. It looks like AFDO versions are up to date." + else + echo "error: update in ${failed_channels} failed" >&2 + exit 3 + fi +fi -- cgit v1.2.3 From 011361fa8f4cdaefa23ecbeb2bc0ff3573b2a403 Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Mon, 24 May 2021 13:47:45 -0700 Subject: update_kernel_afdo: Check last week profiles The script handles Monday differently which make it failing when profiles are not uploaded yet. Instead we need to check profiles updated within the last week so we can safely run the script any day given that profiles are updated weekly. BUG=None TEST=./afdo_tools/update_kernel_afdo Change-Id: I15489e8462c0fbceb9a73c3e3bec3027e26b18d4 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2915677 Reviewed-by: George Burgess Commit-Queue: Denis Nikitin Tested-by: Denis Nikitin --- afdo_tools/update_kernel_afdo | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'afdo_tools/update_kernel_afdo') diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo index aa14b44a..4b41053f 100755 --- a/afdo_tools/update_kernel_afdo +++ b/afdo_tools/update_kernel_afdo @@ -36,13 +36,8 @@ outfile="$(realpath --relative-to="${tc_utils_dir}" \ # Convert toolchain_utils into the absolute path. abs_tc_utils_dir="$(realpath ${tc_utils_dir})" -# The most recent Monday, in Unix timestamp format. -if [ $(date +%a) = "Mon" ] -then - expected_time=$(date +%s -d 00:00:00) -else - expected_time=$(date +%s -d "last Monday") -fi +# Check profiles uploaded within the last week. +expected_time=$(date +%s -d "week ago") declare -A branch branch_number commit remote_repo=$(git -C "${tc_utils_dir}" remote) -- cgit v1.2.3 From 4140da55281d75525f6c429d1e6ebc9332f4f53d Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Wed, 15 Sep 2021 11:12:44 -0700 Subject: update_kernel_afdo: Remove 3.18 Remove 3.18 from the kernel AFDO update. BUG=b:199787442 TEST=./update_kernel_afdo Change-Id: I343d2048fb6bbd9c3c9f37bdfe64526a91c68220 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3162669 Tested-by: Denis Nikitin Reviewed-by: Bob Haarman Commit-Queue: Denis Nikitin --- afdo_tools/update_kernel_afdo | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'afdo_tools/update_kernel_afdo') diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo index 4b41053f..f539ee9d 100755 --- a/afdo_tools/update_kernel_afdo +++ b/afdo_tools/update_kernel_afdo @@ -25,7 +25,7 @@ set -eu set -o pipefail GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel -KVERS="3.18 4.4 4.14 4.19 5.4" +KVERS="4.4 4.14 4.19 5.4" failed_channels="" script_dir=$(dirname "$0") @@ -180,7 +180,6 @@ EOT canary ) commit_contents="afdo_metadata: Publish the new kernel profiles -Update chromeos-kernel-3_18 Update chromeos-kernel-4_4 Update chromeos-kernel-4_14 Update chromeos-kernel-4_19 -- cgit v1.2.3 From 6910c87fa3317ba0d96375b87c6d99f735fbfb7f Mon Sep 17 00:00:00 2001 From: Denis Nikitin Date: Fri, 1 Oct 2021 10:07:39 -0700 Subject: update_kernel_afdo: Skip 95 branch Add a list of skipped chromium branches and adjust canary branch number calculation. BUG=None TEST=`./update_kernel_afdo` shows 96 as canary. Change-Id: If96711f5a0d52c128d4829e338b9b950001c6e75 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3199488 Reviewed-by: Manoj Gupta Tested-by: Denis Nikitin Commit-Queue: Denis Nikitin --- afdo_tools/update_kernel_afdo | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'afdo_tools/update_kernel_afdo') diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo index f539ee9d..ff0ab224 100755 --- a/afdo_tools/update_kernel_afdo +++ b/afdo_tools/update_kernel_afdo @@ -27,6 +27,8 @@ set -o pipefail GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel KVERS="4.4 4.14 4.19 5.4" failed_channels="" +# Add skipped chrome branches in ascending order here. +SKIPPED_BRANCHES="95" script_dir=$(dirname "$0") tc_utils_dir="${script_dir}/.." @@ -57,8 +59,14 @@ branch["canary"]=${canary_ref##*/} # Get current branch numbers (number which goes after R). branch_number["stable"]=$(echo "${branch["stable"]}" | \ sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") -branch_number["beta"]="$((branch_number[stable] + 1))" +branch_number["beta"]=$(echo "${branch["beta"]}" | \ + sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") branch_number["canary"]="$((branch_number[beta] + 1))" +for skipped_branch in $SKIPPED_BRANCHES ; do + if [[ ${branch_number["canary"]} == $skipped_branch ]] ; then + ((branch_number[canary]++)) + fi +done # Without arguments the script updates all branches. channels=${1:-"all"} -- cgit v1.2.3