afdo_tools: add update_kernel_afdo script

This adds a script, update_kernel_afdo, which automatically updates
afdo_metadata/kernel_afdo.json. The changes are then shown with
git diff and can be committed with git commit after review.

BUG=None
TEST=Ran the script, including with some variations to see the failure modes.

Change-Id: Ibe473a80a874e32ddcb9c292e1c018fac72e7292
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2261732
Commit-Queue: Bob Haarman <inglorion@chromium.org>
Tested-by: Bob Haarman <inglorion@chromium.org>
Reviewed-by: Tiancong Wang <tcwang@google.com>

1 files changed, 98 insertions, 0 deletions

diff --git a/afdo_tools/update_kernel_afdo b/afdo_tools/update_kernel_afdo
new file mode 100755
index 00000000..aac891cb
--- /dev/null
+++ b/afdo_tools/update_kernel_afdo
@@ -0,0 +1,98 @@
+#! /bin/sh
+
+# Due to crbug.com/1081332, we need to update AFDO metadata
+# manually. This script performs a few checks and generates a
+# new kernel_afdo.json file, which can then be committed.
+#
+# USAGE:
+# toolchain-utils$ ./afdo_tools/update_kernel_afdo
+#
+# The script modifies the JSON file and shows the git diff.
+#
+# If the changes look good, git commit them. Example commit
+# message (from crrev.com/c/2197462):
+#
+#   afdo_metadata: Publish the new kernel profiles
+#
+#   Update chromeos-kernel-3_18 to R84-13080.0-1589189810
+#   Update chromeos-kernel-4_4 to R84-13080.0-1589189726
+#   Update chromeos-kernel-4_14 to R84-13080.0-1589190025
+#   Update chromeos-kernel-4_19 to R84-13080.0-1589189550
+#
+#   BUG=None
+#   TEST=Verified in kernel-release-afdo-verify-orchestrator.
+#
+
+set -eu
+
+GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel
+KVERS="3.18 4.4 4.14 4.19"
+errs=""
+successes=0
+
+script_dir=$(dirname "$0")
+tc_utils_dir="$script_dir/.."
+metadata_dir="$tc_utils_dir/afdo_metadata"
+outfile="$metadata_dir/kernel_afdo.json"
+
+# The most recent Monday, in Unix timestamp format.
+if [ $(date +%a) = "Mon" ]
+then
+  expected_time=$(date +%s -d 00:00:00)
+else
+  expected_time=$(date +%s -d "last Monday")
+fi
+
+json="{"
+sep=""
+for kver in $KVERS
+do
+  latest=$(gsutil ls -l "$GS_BASE/$kver/" | tail -n 2 | head -n 1)
+
+  # Verify that the file has the expected date.
+  file_time=$(echo "$latest" | awk '{print $2}')
+  file_time_unix=$(date +%s -d "$file_time")
+  if [ $file_time_unix -lt $expected_time ]
+  then
+    expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time)
+    echo "Wrong date for $kver: $file_time is before $expected" >&2
+    errs="$errs $kver"
+    continue
+  fi
+
+  # Generate JSON.
+  json_kver=$(echo "$kver" | tr . _)
+  # b/147370213 (migrating profiles from gcov format) may result in the
+  # pattern below no longer doing the right thing.
+  name=$(echo "$latest" | sed 's%.*/\(.*\)\.gcov.*%\1%')
+  json=$(cat <<EOT
+$json$sep
+    "chromeos-kernel-$json_kver": {
+        "name": "$name"
+    }
+EOT
+)
+  sep=","
+  successes=$((successes + 1))
+done
+
+# If we did not succeed for any kvers, exit now.
+if [ $successes -eq 0 ]
+then
+  echo "error: AFDO profiles out of date for all kernel versions" >&2
+  exit 2
+fi
+
+# Write new JSON file.
+printf "%s\n}\n" "$json" > "$outfile"
+
+# Show the changes.
+(cd "$tc_utils_dir" && git diff)
+
+# If no changes were made, say so.
+outdir=$(dirname "$outfile")
+shortstat=$(cd "$outdir" && git status --short $(basename "$outfile"))
+[ -n "$shortstat" ] || echo $(basename "$outfile")" is up to date."
+
+# If we had any errors, warn about them.
+[ -z "$errs" ] || echo "warning: failed to update$errs" >&2