Merge "Combine both x86/gce and arm flow to one flow. Test: manual atest"

2 files changed, 225 insertions, 0 deletions

diff --git a/tools/create_base_image.go b/tools/create_base_image.go
index 40d533b41..f3adc6b86 100644
--- a/tools/create_base_image.go
+++ b/tools/create_base_image.go
@@ -27,6 +27,7 @@ var dest_image string
 var dest_family string
 var dest_project string
 var launch_instance string
+var arch string
 var source_image_family string
 var source_image_project string
 var repository_url string
@@ -48,17 +49,37 @@ func init() {
     username+"-build", "Instance name to create for the build")
   flag.StringVar(&build_project, "build_project",
     mustShell("gcloud config get-value project"), "Project to use for scratch")
+  // The new get-value output format is different. The result is in 2nd line.
+  str_list := strings.Split(build_project, "\n")
+  if len(str_list) == 2 {
+    build_project = str_list[1]
+  }
+
   flag.StringVar(&build_zone, "build_zone",
     mustShell("gcloud config get-value compute/zone"),
     "Zone to use for scratch resources")
+  // The new get-value output format is different. The result is in 2nd line.
+  str_list = strings.Split(build_zone, "\n")
+  if len(str_list) == 2 {
+    build_zone = str_list[1]
+  }
+
   flag.StringVar(&dest_image, "dest_image",
     "vsoc-host-scratch-"+username, "Image to create")
   flag.StringVar(&dest_family, "dest_family", "",
     "Image family to add the image to")
   flag.StringVar(&dest_project, "dest_project",
     mustShell("gcloud config get-value project"), "Project to use for the new image")
+  // The new get-value output format is different. The result is in 2nd line.
+  str_list = strings.Split(dest_project, "\n")
+  if len(str_list) == 2 {
+    dest_project = str_list[1]
+  }
+
   flag.StringVar(&launch_instance, "launch_instance", "",
     "Name of the instance to launch with the new image")
+  flag.StringVar(&arch, "arch", "gce_x86_64",
+    "Which CPU arch, arm/x86_64/gce_x86_64")
   flag.StringVar(&source_image_family, "source_image_family", "debian-11",
     "Image familty to use as the base")
   flag.StringVar(&source_image_project, "source_image_project", "debian-cloud",
@@ -163,6 +184,71 @@ func main() {
   gpu_type := "nvidia-tesla-p100-vws"
   PZ := "--project=" + build_project + " --zone=" + build_zone
 
+  if arch != "gce_x86_64" {
+    // new path that generate image locally without creating GCE instance
+
+    abt := os.Getenv("ANDROID_BUILD_TOP")
+    cmd := `"` + abt + `/device/google/cuttlefish/tools/create_base_image_combined.sh"`
+    cmd += " " + arch
+    out, err := shell(cmd)
+    if out != "" {
+      fmt.Println(out)
+    }
+    if err != nil {
+      fmt.Println("create_base_image arch %s error occurred: %s", arch, err)
+    }
+
+    // gce operations
+    delete_instances := build_instance + " " + dest_image
+    if launch_instance != "" {
+      delete_instances += " " + launch_instance
+    }
+    zip_file := "disk_"+username+".raw.tar.gz"
+    gs_file := "gs://cloud-android-testing-esp/"+zip_file
+    cloud_storage_file := "https://storage.googleapis.com/cloud-android-testing-esp/"+zip_file
+    location := "us"
+
+    // delete all previous instances, images and disks
+    gce(WarnOnFail, `compute instances delete -q `+PZ+` `+delete_instances, `Not running`)
+    gce(WarnOnFail, `compute disks delete -q `+PZ+` "`+dest_image+`"`, `No scratch disk`)
+    gce(WarnOnFail, `compute images delete -q --project="`+build_project+`" "`+dest_image+`"`,
+      `Not respinning`)
+    gce(WarnOnFail, `alpha storage rm `+gs_file)
+
+    // upload new local host image into GCE storage
+    gce(WarnOnFail, `alpha storage cp `+abt+`/`+zip_file+` gs://cloud-android-testing-esp`)
+
+    // create GCE image based on new uploaded host image
+    gce(WarnOnFail, `compute images create "`+dest_image+`" --project="`+build_project+
+      `" --family="`+source_image_family+`" --source-uri="`+cloud_storage_file+
+      `" --storage-location="`+location+`" --guest-os-features=UEFI_COMPATIBLE`)
+
+    // find Nvidia GPU and then create GCE instance
+    gce(ExitOnFail, `compute accelerator-types describe "`+gpu_type+`" `+PZ,
+      `Please use a zone with `+gpu_type+` GPUs available.`)
+    createInstance(build_instance, PZ+
+      ` --machine-type=n1-standard-16 --network-interface=network-tier=PREMIUM,subnet=default`+
+      ` --accelerator="type=`+gpu_type+
+      `,count=1" --maintenance-policy=TERMINATE --provisioning-model=STANDARD`+
+      ` --service-account=204446994883-compute@developer.gserviceaccount.com`+
+      ` --scopes=https://www.googleapis.com/auth/devstorage.read_only,`+
+      `https://www.googleapis.com/auth/logging.write,`+
+      `https://www.googleapis.com/auth/monitoring.write,`+
+      `https://www.googleapis.com/auth/servicecontrol,`+
+      `https://www.googleapis.com/auth/service.management.readonly,`+
+      `https://www.googleapis.com/auth/trace.append`+
+      ` --tags=http-server --create-disk=auto-delete=yes,boot=yes,device-name=`+build_instance+
+      `,image=projects/cloud-android-testing/global/images/`+dest_image+
+      `,mode=rw,size=200,type=projects/cloud-android-testing/zones/`+build_zone+
+      `/diskTypes/pd-balanced --no-shielded-secure-boot --shielded-vtpm`+
+      ` --shielded-integrity-monitoring --reservation-affinity=any`)
+
+    // enable serial-port (console)
+    gce(WarnOnFail, `compute instances add-metadata `+build_instance+
+      ` --metadata serial-port-enable=TRUE`)
+    return
+  }
+
   dest_family_flag := ""
   if dest_family != "" {
     dest_family_flag = "--family=" + dest_family
diff --git a/tools/create_base_image_combined.sh b/tools/create_base_image_combined.sh
new file mode 100755
index 000000000..b3b20f3e5
--- /dev/null
+++ b/tools/create_base_image_combined.sh
@@ -0,0 +1,139 @@
+#!/bin/bash
+
+# Copyright 2022 Google Inc. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -x
+set -u
+# comment out -e because partx has error message
+# set -o errexit
+# set -e
+
+# get script directory
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+if [ -z $ANDROID_BUILD_TOP ]; then
+  echo "error: run script after 'lunch'"
+  exit 1
+fi
+
+source "${ANDROID_BUILD_TOP}/external/shflags/shflags"
+
+# prepare pre-requested files, including kernel and uboot
+# get temp directory
+tmpdir=`echo $RANDOM | md5sum | head -c 20`
+mkdir "/tmp/$tmpdir"
+UBOOT_REPO="/tmp/$tmpdir/uboot"
+KERNEL_REPO="/tmp/$tmpdir/kernel"
+IMAGE="/tmp/$tmpdir/test_image"
+ARCH=
+FLAGS_HELP="USAGE: $0 [ARCH]"
+mkdir $UBOOT_REPO
+mkdir $KERNEL_REPO
+cd $KERNEL_REPO
+repo init -u persistent-https://android.git.corp.google.com/kernel/manifest -b common-android13-5.15 && repo sync
+
+# parse input parameters
+FLAGS "$@" || exit $?
+eval set -- "${FLAGS_ARGV}"
+
+for arg in "$@" ; do
+  if [ -z $ARCH ]; then
+    ARCH=$arg
+  else
+    flags_help
+    exit 1
+  fi
+done
+
+if [[ "$ARCH" == "arm" ]]; then
+  cd $UBOOT_REPO
+  repo init -u persistent-https://android.git.corp.google.com/kernel/manifest -b u-boot-mainline && repo sync
+fi
+
+if [ -z $KERNEL_REPO -o -z $UBOOT_REPO ]; then
+  flags_help
+  exit 1
+fi
+if [ ! -e "${UBOOT_REPO}" ]; then
+  echo "error: can't find '${UBOOT_REPO}'. aborting..."
+  exit 1
+fi
+if [ ! -e "${KERNEL_REPO}" ]; then
+  echo "error: can't find '${KERNEL_REPO}'. aborting..."
+  exit 1
+fi
+
+tmpfile=`mktemp`
+bootenv=`mktemp`
+cat > ${tmpfile} << "EOF"
+bootdelay=2
+baudrate=1500000
+scriptaddr=0x00500000
+boot_targets=mmc1 mmc0
+bootcmd=run distro_bootcmd
+distro_bootcmd=for target in ${boot_targets}; do run bootcmd_${target}; done
+bootcmd_mmc0=devnum=0; run mmc_boot
+bootcmd_mmc1=devnum=1; run mmc_boot
+mmc_boot=if mmc dev ${devnum}; then ; run scan_for_boot_part; fi
+scan_for_boot_part=part list mmc ${devnum} -bootable devplist; env exists devplist || setenv devplist 1; for distro_bootpart in ${devplist}; do if fstype mmc ${devnum}:${distro_bootpart} bootfstype; then run find_script; fi; done; setenv devplist;
+find_script=if test -e mmc ${devnum}:${distro_bootpart} /boot/boot.scr; then echo Found U-Boot script /boot/boot.scr; run run_scr; fi
+run_scr=load mmc ${devnum}:${distro_bootpart} ${scriptaddr} /boot/boot.scr; source ${scriptaddr}
+EOF
+echo "Sha=`${script_dir}/gen_sha.sh --uboot ${UBOOT_REPO} --kernel ${KERNEL_REPO}`" >> ${tmpfile}
+${ANDROID_BUILD_TOP}/device/google/cuttlefish_prebuilts/uboot_tools/mkenvimage -s 32768 -o ${bootenv} - < ${tmpfile}
+
+# build uboot based on architecture
+cd ${UBOOT_REPO}
+if [[ "$ARCH" == "arm" ]]; then
+  BUILD_CONFIG=u-boot/build.config.rockpi4 build/build.sh -j1
+fi
+cd -
+
+# build kernel based on architecture
+cd ${KERNEL_REPO}
+rm -rf out
+if [[ "$ARCH" == "arm" ]]; then
+  BUILD_CONFIG=common/build.config.rockpi4 build/build.sh -j`nproc`
+else
+  BUILD_CONFIG=common/build.config.gce.x86_64 build/build.sh -j`nproc`
+fi
+cd -
+
+dist_dir=$(echo ${KERNEL_REPO}/out/android*/dist)
+
+# build rootfs/host images
+if [[ "$ARCH" == "arm" ]]; then
+  ${ANDROID_BUILD_TOP}/kernel/tests/net/test/build_rootfs.sh \
+    -a arm64 -s bullseye-rockpi -n ${IMAGE} -r ${IMAGE}.initrd -e \
+    -k ${dist_dir}/Image -i ${dist_dir}/initramfs.img \
+    -d ${dist_dir}/rk3399-rock-pi-4b.dtb:rockchip
+else
+  ${ANDROID_BUILD_TOP}/kernel/tests/net/test/build_rootfs.sh \
+    -a amd64 -s bullseye-server -n ${IMAGE} -r ${IMAGE}.initrd -e \
+    -k ${dist_dir}/bzImage -i ${dist_dir}/initramfs.img -g
+fi
+
+if [ $? -ne 0 ]; then
+  echo "error: failed to build rootfs. exiting..."
+  exit 1
+fi
+
+rm -f ${IMAGE}.initrd
+rm -rf ${ANDROID_BUILD_TOP}/disk.raw
+cp ${IMAGE} ${ANDROID_BUILD_TOP}/disk.raw
+cd ${ANDROID_BUILD_TOP}
+rm -rf disk_${USER}.raw.tar.gz
+tar Szcvf disk_${USER}.raw.tar.gz disk.raw