diff options
Diffstat (limited to 'afdo_tools/update_kernel_afdo')
| -rwxr-xr-x | afdo_tools/update_kernel_afdo | 308 |
1 files changed, 210 insertions, 98 deletions
diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo index ff0ab224..9e4d645d 100755 --- a/afdo_tools/update_kernel_afdo +++ b/afdo_tools/update_kernel_afdo @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Copyright 2020 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,37 +9,74 @@ # USAGE=" -Usage: $(basename $0) [main|beta|stable|all] [--help] +Usage: $(basename "$0") [--noupload|-upload] [main|beta|stable|all] [--help] Description: The script takes one optional argument which is the channel where we want to update the kernel afdo and creates a commit (or commits with \"all\" channels) in the corresponding branch. No arguments defaults to \"all\". - Follow the prompt to submit the changes. + Follow the prompt to upload the changes. NO CLEAN-UP NEEDED. The script ignores any local changes and keeps the current branch unchanged. + + Args: + --help Show this help. + --upload Upload CLs when the update succeeded (default). + --noupload Do not upload CLs. Instead, print the upload commands. + main|beta|stable Update metadata only on the specified channel. " set -eu set -o pipefail -GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel -KVERS="4.4 4.14 4.19 5.4" +AMD_GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel +ARM_GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel/arm +AMD_KVERS="4.14 4.19 5.4 5.10" +ARM_KVERS="5.15" failed_channels="" # Add skipped chrome branches in ascending order here. SKIPPED_BRANCHES="95" +# NOTE: We enable/disable kernel AFDO starting from a particular branch. +# For example if we want to enable kernel AFDO in 5.15, first, we do it +# in main. In this case we want to disable it in beta and stable branches. +# The second scenario is when we want to disable kernel AFDO (when all devices +# move to kernelnext and there are no new profiles from the field). In this +# case we disable AFDO in main but still keep it live in beta and stable. +declare -A SKIPPED_KVERS_IN_BRANCHES +# In SKIPPED_KVERS_IN_BRANCHES +# - key is a branch number string; +# - value is the list of kernels separated by space. +# Example: SKIPPED_KVERS_IN_BRANCHES["105"]="4.4 4.14" + +# b/223115767. In M-100 there are no new profiles in 5.10. And AFDO is not +# enabled on any 5.10 board in M-100 either. +SKIPPED_KVERS_IN_BRANCHES["100"]="5.10" + script_dir=$(dirname "$0") tc_utils_dir="${script_dir}/.." metadata_dir="${tc_utils_dir}/afdo_metadata" -outfile="$(realpath --relative-to="${tc_utils_dir}" \ +amd_outfile="$(realpath --relative-to="${tc_utils_dir}" \ "${metadata_dir}"/kernel_afdo.json)" +arm_outfile="$(realpath --relative-to="${tc_utils_dir}" \ + "${metadata_dir}"/kernel_arm_afdo.json)" # Convert toolchain_utils into the absolute path. -abs_tc_utils_dir="$(realpath ${tc_utils_dir})" +abs_tc_utils_dir="$(realpath "${tc_utils_dir}")" # Check profiles uploaded within the last week. expected_time=$(date +%s -d "week ago") +# Upload CLs on success. +upload_cl=true + +ARCHS="amd arm" +declare -A arch_gsbase arch_kvers arch_outfile +arch_gsbase["amd"]="${AMD_GS_BASE}" +arch_gsbase["arm"]="${ARM_GS_BASE}" +arch_kvers["amd"]="${AMD_KVERS}" +arch_kvers["arm"]="${ARM_KVERS}" +arch_outfile["amd"]="${amd_outfile}" +arch_outfile["arm"]="${arm_outfile}" declare -A branch branch_number commit remote_repo=$(git -C "${tc_utils_dir}" remote) @@ -47,10 +84,9 @@ canary_ref="refs/heads/main" # Read the last two release-Rxx from remote branches # and assign them to stable_ref and beta_ref. # sort -V is the version sort which puts R100 after R99. -last_branches=$(git -C "${tc_utils_dir}" ls-remote -h "${remote_repo}" \ - release-R\* | cut -f2 | sort -V | tail -n 2) # We need `echo` to convert newlines into spaces for read. -read stable_ref beta_ref <<< $(echo ${last_branches}) +read -r stable_ref beta_ref <<< "$(git -C "${tc_utils_dir}" ls-remote -h \ + "${remote_repo}" release-R\* | cut -f2 | sort -V | tail -n 2 | paste -s)" # Branch names which start from release-R. branch["beta"]=${beta_ref##*/} branch["stable"]=${stable_ref##*/} @@ -62,33 +98,53 @@ branch_number["stable"]=$(echo "${branch["stable"]}" | \ branch_number["beta"]=$(echo "${branch["beta"]}" | \ sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") branch_number["canary"]="$((branch_number[beta] + 1))" -for skipped_branch in $SKIPPED_BRANCHES ; do - if [[ ${branch_number["canary"]} == $skipped_branch ]] ; then +for skipped_branch in ${SKIPPED_BRANCHES} ; do + if [[ ${branch_number["canary"]} == "${skipped_branch}" ]] ; then ((branch_number[canary]++)) fi done # Without arguments the script updates all branches. -channels=${1:-"all"} -case "${channels}" in +channels="" +for arg in "$@" +do + case "${arg}" in stable | canary | beta ) + channels="${channels} ${arg}" ;; main ) - channels="canary" + channels="${channels} canary" ;; all ) channels="canary beta stable" ;; + --noupload | --no-upload) + upload_cl=false + ;; + --upload) + upload_cl=true + ;; --help | help | -h ) - echo "$USAGE" + echo "${USAGE}" exit 0 ;; - * ) - echo "Channel \"${channels}\" is not supported. + -*) + echo "Option \"${arg}\" is not supported." >&2 + echo "${USAGE}" + exit 1 + ;; + *) + echo "Channel \"${arg}\" is not supported. Must be main (or canary), beta, stable or all." >&2 - echo "$USAGE" + echo "${USAGE}" exit 1 -esac + esac +done + +if [[ -z "${channels}" ]] +then + channels="canary beta stable" +fi # Fetch latest branches. git -C "${tc_utils_dir}" fetch "${remote_repo}" @@ -99,11 +155,20 @@ echo "-> Working in ${worktree_dir}" # This way we don't need to clean-up and sync toolchain_utils before the # change. Neither we should care about clean-up after the submit. git -C "${tc_utils_dir}" worktree add --detach "${worktree_dir}" -trap "git -C ${abs_tc_utils_dir} worktree remove ${worktree_dir}" EXIT -cd "${worktree_dir}" +trap 'git -C "${abs_tc_utils_dir}" worktree remove -f "${worktree_dir}"' EXIT +pushd "${worktree_dir}" for channel in ${channels} do + set +u + if [[ -n "${commit[${channel}]}" ]] + then + echo "Skipping channel ${channel} which already has commit\ + ${commit[${channel}]}." + continue + fi + set -u + errs="" successes=0 curr_branch_number=${branch_number[${channel}]} @@ -111,87 +176,124 @@ do echo echo "Checking \"${channel}\" channel..." echo "branch_number=${curr_branch_number} branch=${curr_branch}" - json="{" - sep="" - for kver in $KVERS + + git reset --hard HEAD + git checkout "${remote_repo}/${curr_branch}" + + for arch in ${ARCHS} do - # Sort the gs output by timestamp (default ordering is by name, so - # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz) - latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ - grep "R${curr_branch_number}" | tail -1 || true) - if [[ -z "$latest" && "${channel}" != "stable" ]] - then - # if no profiles exist for the current branch, try the previous branch - latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ - grep "R$((curr_branch_number - 1))" | tail -1) - fi + json="{" + sep="" + for kver in ${arch_kvers[${arch}]} + do + # Skip kernels disabled in this branch. + skipped=false + for skipped_branch in "${!SKIPPED_KVERS_IN_BRANCHES[@]}" + do + if [[ ${curr_branch_number} == "${skipped_branch}" ]] + then + # Current branch is in the keys of SKIPPED_KVERS_IN_BRANCHES. + # Now lets check if $kver is in the list. + for skipped_kver in ${SKIPPED_KVERS_IN_BRANCHES[${skipped_branch}]} + do + if [[ ${kver} == "${skipped_kver}" ]] + then + skipped=true + break + fi + done + fi + done + if ${skipped} + then + echo "${kver} is skipped in branch ${curr_branch_number}. Skip it." + continue + fi + # Sort the gs output by timestamp, default ordering is by name. So + # R86-13310.3-1594633089.gcov.xz goes after + # R86-13310.18-1595237847.gcov.xz. + latest=$(gsutil.py ls -l "${arch_gsbase[${arch}]}/${kver}/" | sort -k2 | \ + grep "R${curr_branch_number}" | tail -1 || true) + if [[ -z "${latest}" && "${channel}" != "stable" ]] + then + # if no profiles exist for the current branch, try the previous branch + latest=$(gsutil.py ls -l "${arch_gsbase[${arch}]}/${kver}/" | \ + sort -k2 | grep "R$((curr_branch_number - 1))" | tail -1) + fi - # Verify that the file has the expected date. - file_time=$(echo "$latest" | awk '{print $2}') - file_time_unix=$(date +%s -d "$file_time") - if [ $file_time_unix -lt $expected_time ] - then - expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time) - echo "Wrong date for $kver: $file_time is before $expected" >&2 - errs="$errs $kver" - continue - fi + # Verify that the file has the expected date. + file_time=$(echo "${latest}" | awk '{print $2}') + file_time_unix=$(date +%s -d "${file_time}") + if [ "${file_time_unix}" -lt "${expected_time}" ] + then + expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @"${expected_time}") + echo "Wrong date for ${kver}: ${file_time} is before ${expected}" >&2 + errs="${errs} ${kver}" + continue + fi - # Generate JSON. - json_kver=$(echo "$kver" | tr . _) - # b/147370213 (migrating profiles from gcov format) may result in the - # pattern below no longer doing the right thing. - name=$(echo "$latest" | sed 's%.*/\(.*\)\.gcov.*%\1%') - json=$(cat <<EOT -$json$sep - "chromeos-kernel-$json_kver": { - "name": "$name" + # Generate JSON. + json_kver=$(echo "${kver}" | tr . _) + # b/147370213 (migrating profiles from gcov format) may result in the + # pattern below no longer doing the right thing. + name="$(basename "${latest%.gcov.*}")" + # Skip kernels with no AFDO support in the current channel. + if [[ "${name}" == "" ]] + then + continue + fi + json=$(cat <<EOT +${json}${sep} + "chromeos-kernel-${json_kver}": { + "name": "${name}" } EOT - ) - sep="," - successes=$((successes + 1)) - done + ) + sep="," + successes=$((successes + 1)) + done # kvers loop - # If we did not succeed for any kvers, exit now. - if [[ $successes -eq 0 ]] - then - echo "error: AFDO profiles out of date for all kernel versions" >&2 - failed_channels="${failed_channels} ${channel}" - continue - fi + # If we did not succeed for any kvers, exit now. + if [[ ${successes} -eq 0 ]] + then + echo "error: AFDO profiles out of date for all kernel versions" >&2 + failed_channels="${failed_channels} ${channel}" + continue + fi - git reset --hard HEAD - echo git checkout "${remote_repo}/${curr_branch}" - git checkout "${remote_repo}/${curr_branch}" + # Write new JSON file. + # Don't use `echo` since `json` might have esc characters in it. + printf "%s\n}\n" "${json}" > "${arch_outfile[${arch}]}" - # Write new JSON file. - # Don't use `echo` since `json` might have esc characters in it. - printf "%s\n}\n" "$json" > "$outfile" + # If no changes were made, say so. + outdir=$(dirname "${arch_outfile[${arch}]}") + shortstat=$(cd "${outdir}" &&\ + git status --short "$(basename "${arch_outfile[${arch}]}")") + [ -z "${shortstat}" ] &&\ + echo "$(basename "${arch_outfile[${arch}]}") is up to date." \ + && continue - # If no changes were made, say so. - outdir=$(dirname "$outfile") - shortstat=$(cd "$outdir" && git status --short $(basename "$outfile")) - [ -z "$shortstat" ] && echo $(basename "$outfile")" is up to date." \ - && continue + # If we had any errors, warn about them. + if [[ -n "${errs}" ]] + then + echo "warning: failed to update ${errs} in ${channel}" >&2 + failed_channels="${failed_channels} ${channel}" + continue + fi - # If we had any errors, warn about them. - if [[ -n "$errs" ]] - then - echo "warning: failed to update $errs in ${channel}" >&2 - failed_channels="${failed_channels} ${channel}" - continue - fi + git add "${arch_outfile[${arch}]}" + done # ARCHS loop - git add afdo_metadata/kernel_afdo.json case "${channel}" in canary ) - commit_contents="afdo_metadata: Publish the new kernel profiles - -Update chromeos-kernel-4_4 -Update chromeos-kernel-4_14 -Update chromeos-kernel-4_19 -Update chromeos-kernel-5_4 + commit_contents=$'afdo_metadata: Publish the new kernel profiles\n\n' + for arch in ${ARCHS} ; do + for kver in ${arch_kvers[${arch}]} ; do + commit_contents="${commit_contents}Update ${arch} profile on\ + chromeos-kernel-${kver}"$'\n' + done + done + commit_contents="${commit_contents} BUG=None TEST=Verified in kernel-release-afdo-verify-orchestrator" @@ -215,20 +317,30 @@ TEST=Verified in kernel-release-afdo-verify-orchestrator" commit[${channel}]=$(git -C "${worktree_dir}" rev-parse HEAD) done +popd echo # Array size check doesn't play well with the unbound variable option. set +u if [[ ${#commit[@]} -gt 0 ]] then set -u - echo "The change is applied in ${!commit[@]}." - echo "Run these commands to submit the change:" - echo - for channel in ${!commit[@]} - do - echo -e "\tgit -C ${tc_utils_dir} push ${remote_repo} \ -${commit[${channel}]}:refs/for/${branch[${channel}]}" - done + echo "The change is applied in ${!commit[*]}." + if ${upload_cl} + then + for channel in "${!commit[@]}" + do + git -C "${tc_utils_dir}" push "${remote_repo}" \ + "${commit[${channel}]}:refs/for/${branch[${channel}]}" + done + else + echo "Run these commands to upload the change:" + echo + for channel in "${!commit[@]}" + do + echo -e "\tgit -C ${tc_utils_dir} push ${remote_repo} \ + ${commit[${channel}]}:refs/for/${branch[${channel}]}" + done + fi # Report failed channels. if [[ -n "${failed_channels}" ]] |