From 4fd94cfcf97abc6cb75f49359aba4b2ec01e8b1e Mon Sep 17 00:00:00 2001
From: Neil Fuller <nfuller@google.com>
Date: Thu, 16 Apr 2020 14:01:10 +0100
Subject: Android additions to timezone-boundary-builder

Android additions to timezone-boundary-builder to assist with the data
generation process execution.

Addition of some Docker infra to improve repeatability of execution
(not improve security). Also a README.md to explain the process.

Note: Use --skip_analyze_diffs for run_tzdb.sh as the analysis doesn't
work well with --dist_dir. Upstream fix to follow.

Bug: 190733142
Test: Manual testing
Change-Id: I7b6d54e8dc56a03444d4b71fe3891384a87ce2d6
---
 android/README.md                              |  69 ++++++++++++++++
 android/tools/container/run_tzbb.sh            |  27 +++++++
 android/tools/host/Dockerfile                  |  15 ++++
 android/tools/host/clear_android_data_files.sh |  35 ++++++++
 android/tools/host/common.sh                   |  37 +++++++++
 android/tools/host/download_input_files.sh     | 106 +++++++++++++++++++++++++
 android/tools/host/run_process_in_docker.sh    |  30 +++++++
 android/tools/host/run_tzbb.sh                 |  25 ++++++
 8 files changed, 344 insertions(+)
 create mode 100644 android/README.md
 create mode 100755 android/tools/container/run_tzbb.sh
 create mode 100644 android/tools/host/Dockerfile
 create mode 100755 android/tools/host/clear_android_data_files.sh
 create mode 100755 android/tools/host/common.sh
 create mode 100755 android/tools/host/download_input_files.sh
 create mode 100755 android/tools/host/run_process_in_docker.sh
 create mode 100755 android/tools/host/run_tzbb.sh

diff --git a/android/README.md b/android/README.md
new file mode 100644
index 0000000..62e440a
--- /dev/null
+++ b/android/README.md
@@ -0,0 +1,69 @@
+This directory contains Android's additions to timezone-boundary-builder
+project.
+
+Directories:
+
+`android_inputs/`
+  - Files downloaded from upstream to seed `android_downloads/`
+
+`android_downloads/`
+  - The .json files used to generate the output. Equivalent to upstream's
+    `downloads/` directory.
+
+`android_dist/`
+  - The generated output files. Equivalent to upstream's `dist/` directory.
+
+`tools/`
+  - Scripts used to execute the timezone-boundary-builder generation process.
+
+Android initially intends to use the time zone boundary file, `combined.json`,
+released by upstream. Therefore, this copy of timezone-boundary-builder may be
+unnecessary until / unless Android wants to make local modifications.
+
+-----
+
+The `combined.json` generation process:
+
+Execute `android/tools/host/clear_android_data_files.sh` to clear out most
+working files and reset the local environment.
+
+Before running the generation process, decide if you want to use the same input
+files as have been used by upstream.
+
+Upstream make the files used available in a file called `input-files.zip`, which
+can be downloaded and unpacked into the `android/android_downloads/` directory
+using `android/tools/host/download_input_files.sh`.
+
+When not using upstream's input files, the generation process will first
+download boundary data from OpenStreetMap, which is slow. Because OpenStreetMap
+is constantly evolving, the latest features downloaded will usually have issues
+that need to be fixed before proceeding. The script can be restarted multiple
+times and will usually restart downloads from where it failed last time.
+
+Local edits to input files may be necessary to make adjustments needed for
+Android in future.
+
+timezone-boundary-builder requires the nodejs runtime for execution with various
+dependencies, so Android's `android/tools/host/run_tzbb.sh` uses a Docker
+container to setup the runtime environment in a repeatable manner.
+
+Execute `android/tools/host/run_tzbb.sh --help` to see options.
+
+The following can be used to speed up generation times:
+
+```
+      --skip_analyze_diffs  Skip analysis of diffs between versions    [boolean]
+      --skip_shapefile      Skip shapefile creation                    [boolean]
+      --skip_validation     Skip validation                            [boolean]
+```
+
+After executing `android/tools/host/run_tzdbb.sh`, in `android/android_dist/`
+there will be many .json files alongside (a newly generated) `input-files.zip`
+and `combined.json` that constitute the main output files.
+
+The contents of `android/android_downloads/` and `android/android_dist/` can be
+committed to form a full record of how the latest output file was generated.
+
+The `combined.json` and `input-files.zip` can be passed to the new step of the
+reference data generation pipeline, which is held in a separate git project.
+
diff --git a/android/tools/container/run_tzbb.sh b/android/tools/container/run_tzbb.sh
new file mode 100755
index 0000000..5e4d9da
--- /dev/null
+++ b/android/tools/container/run_tzbb.sh
@@ -0,0 +1,27 @@
+#!/bin/bash -x
+
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -z $DOCKER ]; then
+  echo Run this inside the docker container.
+  exit 1;
+fi
+
+cd /timezone-boundary-builder
+npm root
+npm install
+npm audit fix
+
+node --max-old-space-size=8192 index.js $*
diff --git a/android/tools/host/Dockerfile b/android/tools/host/Dockerfile
new file mode 100644
index 0000000..72e0d79
--- /dev/null
+++ b/android/tools/host/Dockerfile
@@ -0,0 +1,15 @@
+FROM node:12
+ARG userid
+ARG groupid
+ARG username
+ARG tzbbroot
+
+RUN apt-get update && apt-get install -y gdal-bin zip
+
+# Set up the user so that files are owned by the proper user
+RUN groupadd -g $groupid $username \
+ && useradd -m -u $userid -g $groupid $username
+RUN mkdir -p $tzbbroot && chown $userid $tzbbroot
+ENV DOCKER=true
+USER $username
+
diff --git a/android/tools/host/clear_android_data_files.sh b/android/tools/host/clear_android_data_files.sh
new file mode 100755
index 0000000..be59d8b
--- /dev/null
+++ b/android/tools/host/clear_android_data_files.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail fast on any error.
+set -e
+
+HOST_TOOLS_DIR=$(realpath $(dirname $0))
+source ${HOST_TOOLS_DIR}/common.sh
+
+DIRS=(\
+  ${HOST_INPUTS_DIR}\
+  ${HOST_DOWNLOADS_DIR}\
+  ${HOST_DIST_DIR}\
+)
+
+for DIR in ${DIRS[@]}; do
+  echo Deleting content of ${DIR}
+  rm -f ${DIR}/*
+  # Just in case
+  mkdir -p ${DIR}
+done
+
diff --git a/android/tools/host/common.sh b/android/tools/host/common.sh
new file mode 100755
index 0000000..cf9896a
--- /dev/null
+++ b/android/tools/host/common.sh
@@ -0,0 +1,37 @@
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ -z "${HOST_TOOLS_DIR}" ]]; then
+  echo HOST_TOOLS_DIR not set
+  exit 1
+fi
+
+# The android_ prefix is to get around the upstream project's .gitignore rules
+# for dist / downloads. Android will want to commit them to version control to
+# improve change tracking / repeatability and avoid unnecessary load on OSM
+# servers.
+INPUTS_DIR=android_inputs
+DOWNLOADS_DIR=android_downloads
+DIST_DIR=android_dist
+
+CONTAINER_ANDROID_DIR=./android
+CONTAINER_DOWNLOADS_DIR=${CONTAINER_ANDROID_DIR}/${DOWNLOADS_DIR}
+CONTAINER_DIST_DIR=${CONTAINER_ANDROID_DIR}/${DIST_DIR}
+
+HOST_ANDROID_DIR=${HOST_TOOLS_DIR}/../..
+HOST_ANDROID_DIR=$(realpath ${HOST_ANDROID_DIR})
+HOST_INPUTS_DIR=${HOST_ANDROID_DIR}/${INPUTS_DIR}
+HOST_DOWNLOADS_DIR=${HOST_ANDROID_DIR}/${DOWNLOADS_DIR}
+HOST_DIST_DIR=${HOST_ANDROID_DIR}/${DIST_DIR}
+
diff --git a/android/tools/host/download_input_files.sh b/android/tools/host/download_input_files.sh
new file mode 100755
index 0000000..821c6b3
--- /dev/null
+++ b/android/tools/host/download_input_files.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail fast on any error.
+set -e
+
+HOST_TOOLS_DIR=$(realpath $(dirname $0))
+source ${HOST_TOOLS_DIR}/common.sh
+
+RELEASE_ID=${1}
+if [ -z ${RELEASE_ID} ]; then
+  echo "Usage:"
+  echo "  ${0} <release ID>"
+  echo
+  echo "e.g. ${0} 2020d"
+  exit 1
+fi
+
+SCRIPT_PATH=$(realpath $0)
+
+# -f = report failures
+# -L = follow redirects
+CURL_CMD="curl -f -L"
+
+TZBB_PROJECT_URL=https://github.com/evansiroky/timezone-boundary-builder/
+SERVER_RELEASE_BASE_URL=${TZBB_PROJECT_URL}releases/download
+SERVER_SOURCE_BASE_URL=https://raw.githubusercontent.com/evansiroky/timezone-boundary-builder/master
+
+INPUT_DATA_FILE=input-data.zip
+LICENSE_FILE=DATA_LICENSE
+
+function download() {
+  set -e
+  SERVER_BASE_URL=${1}
+  FILE_NAME=${2}
+  LOCAL_DIR=${3}
+
+  URL=${SERVER_BASE_URL}/${FILE_NAME}
+  LOCAL_FILE_NAME=${LOCAL_DIR}/${FILE_NAME}
+  echo Downloading ${FILE_NAME} from ${URL}...
+  ${CURL_CMD} ${URL} --output ${LOCAL_FILE_NAME}
+
+  DOWNLOAD_DATE=$(date)
+  SHA=$(sha1sum ${LOCAL_FILE_NAME} | awk '{ print $1}')
+
+  # Add a METADATA file URL entry for every file we download.
+  cat << EOF >> ${METADATA_FILE}
+  url {
+    type: OTHER
+    value: "${FILE_NAME} downloaded from ${URL} on ${DOWNLOAD_DATE}, SHA1=${SHA}"
+  }
+EOF
+}
+
+mkdir -p ${HOST_INPUTS_DIR}
+echo Removing existing data files...
+rm -f ${HOST_INPUTS_DIR}/*
+
+METADATA_FILE=${HOST_INPUTS_DIR}/METADATA
+# Start the METADATA file
+cat << EOF > ${METADATA_FILE}
+// Generated file DO NOT EDIT
+// Run download_input_files.sh
+name: "android_inputs"
+description:
+    "Assorted files downloaded from ${TZBB_PROJECT_URL}"
+
+third_party {
+  url {
+    type: HOMEPAGE
+    value: "${TZBB_PROJECT_URL}"
+  }
+EOF
+
+download ${SERVER_SOURCE_BASE_URL} ${LICENSE_FILE} ${HOST_INPUTS_DIR}
+download ${SERVER_RELEASE_BASE_URL}/${RELEASE_ID} ${INPUT_DATA_FILE} ${HOST_INPUTS_DIR}
+
+PROTO_UPGRADE_DATE=$(date +'{ year: '%Y' month: '%-m' day: '%-d' }')
+
+# Finish the METADATA file
+cat << EOF >> ${METADATA_FILE}
+  version: "${RELEASE_ID}"
+  last_upgrade_date ${PROTO_UPGRADE_DATE}
+  license_type: RESTRICTED
+}
+EOF
+
+ln -sr ${HOST_INPUTS_DIR}/DATA_LICENSE ${HOST_INPUTS_DIR}/LICENSE
+
+# Handle the unzip step to HOST_DOWNLOADS_DIR
+unzip -d ${HOST_DOWNLOADS_DIR} ${HOST_INPUTS_DIR}/${INPUT_DATA_FILE}
+
+echo Look in ${HOST_INPUTS_DIR} for input files....
diff --git a/android/tools/host/run_process_in_docker.sh b/android/tools/host/run_process_in_docker.sh
new file mode 100755
index 0000000..bdb615a
--- /dev/null
+++ b/android/tools/host/run_process_in_docker.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOST_TOOLS_DIR=$(realpath $(dirname $0))
+CONTAINER_TOOLS_DIR=$(realpath $HOST_TOOLS_DIR/../container)
+TZBB_ROOT=$(realpath $HOST_TOOLS_DIR/../../..)
+
+DOCKER_USERNAME=$(id -un)
+DOCKER_UID=$(id -u)
+DOCKER_GID=$(id -g)
+
+echo "This may need your sudo password in order to access docker:"
+set -x
+sudo docker build --build-arg userid=$DOCKER_UID --build-arg groupid=$DOCKER_GID --build-arg username=$DOCKER_USERNAME --build-arg tzbbroot=$TZBB_ROOT -t android-tzbb .
+sudo docker run -it --rm -v $TZBB_ROOT:/timezone-boundary-builder android-tzbb $*
+set +x
+
diff --git a/android/tools/host/run_tzbb.sh b/android/tools/host/run_tzbb.sh
new file mode 100755
index 0000000..4c30453
--- /dev/null
+++ b/android/tools/host/run_tzbb.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOST_TOOLS_DIR=$(realpath $(dirname $0))
+source ${HOST_TOOLS_DIR}/common.sh
+
+${HOST_TOOLS_DIR}/run_process_in_docker.sh /timezone-boundary-builder/android/tools/container/run_tzbb.sh \
+  --downloads_dir ${CONTAINER_DOWNLOADS_DIR} \
+  --dist_dir ${CONTAINER_DIST_DIR} \
+  --skip_zip \
+  --skip_shapefile \
+  $*
-- 
cgit v1.2.3