From 4fd94cfcf97abc6cb75f49359aba4b2ec01e8b1e Mon Sep 17 00:00:00 2001 From: Neil Fuller Date: Thu, 16 Apr 2020 14:01:10 +0100 Subject: Android additions to timezone-boundary-builder Android additions to timezone-boundary-builder to assist with the data generation process execution. Addition of some Docker infra to improve repeatability of execution (not improve security). Also a README.md to explain the process. Note: Use --skip_analyze_diffs for run_tzdb.sh as the analysis doesn't work well with --dist_dir. Upstream fix to follow. Bug: 190733142 Test: Manual testing Change-Id: I7b6d54e8dc56a03444d4b71fe3891384a87ce2d6 --- android/tools/host/Dockerfile | 15 ++++ android/tools/host/clear_android_data_files.sh | 35 ++++++++ android/tools/host/common.sh | 37 +++++++++ android/tools/host/download_input_files.sh | 106 +++++++++++++++++++++++++ android/tools/host/run_process_in_docker.sh | 30 +++++++ android/tools/host/run_tzbb.sh | 25 ++++++ 6 files changed, 248 insertions(+) create mode 100644 android/tools/host/Dockerfile create mode 100755 android/tools/host/clear_android_data_files.sh create mode 100755 android/tools/host/common.sh create mode 100755 android/tools/host/download_input_files.sh create mode 100755 android/tools/host/run_process_in_docker.sh create mode 100755 android/tools/host/run_tzbb.sh (limited to 'android/tools/host') diff --git a/android/tools/host/Dockerfile b/android/tools/host/Dockerfile new file mode 100644 index 0000000..72e0d79 --- /dev/null +++ b/android/tools/host/Dockerfile @@ -0,0 +1,15 @@ +FROM node:12 +ARG userid +ARG groupid +ARG username +ARG tzbbroot + +RUN apt-get update && apt-get install -y gdal-bin zip + +# Set up the user so that files are owned by the proper user +RUN groupadd -g $groupid $username \ + && useradd -m -u $userid -g $groupid $username +RUN mkdir -p $tzbbroot && chown $userid $tzbbroot +ENV DOCKER=true +USER $username + diff --git a/android/tools/host/clear_android_data_files.sh b/android/tools/host/clear_android_data_files.sh new file mode 100755 index 0000000..be59d8b --- /dev/null +++ b/android/tools/host/clear_android_data_files.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fail fast on any error. +set -e + +HOST_TOOLS_DIR=$(realpath $(dirname $0)) +source ${HOST_TOOLS_DIR}/common.sh + +DIRS=(\ + ${HOST_INPUTS_DIR}\ + ${HOST_DOWNLOADS_DIR}\ + ${HOST_DIST_DIR}\ +) + +for DIR in ${DIRS[@]}; do + echo Deleting content of ${DIR} + rm -f ${DIR}/* + # Just in case + mkdir -p ${DIR} +done + diff --git a/android/tools/host/common.sh b/android/tools/host/common.sh new file mode 100755 index 0000000..cf9896a --- /dev/null +++ b/android/tools/host/common.sh @@ -0,0 +1,37 @@ +# Copyright 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "${HOST_TOOLS_DIR}" ]]; then + echo HOST_TOOLS_DIR not set + exit 1 +fi + +# The android_ prefix is to get around the upstream project's .gitignore rules +# for dist / downloads. Android will want to commit them to version control to +# improve change tracking / repeatability and avoid unnecessary load on OSM +# servers. +INPUTS_DIR=android_inputs +DOWNLOADS_DIR=android_downloads +DIST_DIR=android_dist + +CONTAINER_ANDROID_DIR=./android +CONTAINER_DOWNLOADS_DIR=${CONTAINER_ANDROID_DIR}/${DOWNLOADS_DIR} +CONTAINER_DIST_DIR=${CONTAINER_ANDROID_DIR}/${DIST_DIR} + +HOST_ANDROID_DIR=${HOST_TOOLS_DIR}/../.. +HOST_ANDROID_DIR=$(realpath ${HOST_ANDROID_DIR}) +HOST_INPUTS_DIR=${HOST_ANDROID_DIR}/${INPUTS_DIR} +HOST_DOWNLOADS_DIR=${HOST_ANDROID_DIR}/${DOWNLOADS_DIR} +HOST_DIST_DIR=${HOST_ANDROID_DIR}/${DIST_DIR} + diff --git a/android/tools/host/download_input_files.sh b/android/tools/host/download_input_files.sh new file mode 100755 index 0000000..821c6b3 --- /dev/null +++ b/android/tools/host/download_input_files.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Copyright 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fail fast on any error. +set -e + +HOST_TOOLS_DIR=$(realpath $(dirname $0)) +source ${HOST_TOOLS_DIR}/common.sh + +RELEASE_ID=${1} +if [ -z ${RELEASE_ID} ]; then + echo "Usage:" + echo " ${0} " + echo + echo "e.g. ${0} 2020d" + exit 1 +fi + +SCRIPT_PATH=$(realpath $0) + +# -f = report failures +# -L = follow redirects +CURL_CMD="curl -f -L" + +TZBB_PROJECT_URL=https://github.com/evansiroky/timezone-boundary-builder/ +SERVER_RELEASE_BASE_URL=${TZBB_PROJECT_URL}releases/download +SERVER_SOURCE_BASE_URL=https://raw.githubusercontent.com/evansiroky/timezone-boundary-builder/master + +INPUT_DATA_FILE=input-data.zip +LICENSE_FILE=DATA_LICENSE + +function download() { + set -e + SERVER_BASE_URL=${1} + FILE_NAME=${2} + LOCAL_DIR=${3} + + URL=${SERVER_BASE_URL}/${FILE_NAME} + LOCAL_FILE_NAME=${LOCAL_DIR}/${FILE_NAME} + echo Downloading ${FILE_NAME} from ${URL}... + ${CURL_CMD} ${URL} --output ${LOCAL_FILE_NAME} + + DOWNLOAD_DATE=$(date) + SHA=$(sha1sum ${LOCAL_FILE_NAME} | awk '{ print $1}') + + # Add a METADATA file URL entry for every file we download. + cat << EOF >> ${METADATA_FILE} + url { + type: OTHER + value: "${FILE_NAME} downloaded from ${URL} on ${DOWNLOAD_DATE}, SHA1=${SHA}" + } +EOF +} + +mkdir -p ${HOST_INPUTS_DIR} +echo Removing existing data files... +rm -f ${HOST_INPUTS_DIR}/* + +METADATA_FILE=${HOST_INPUTS_DIR}/METADATA +# Start the METADATA file +cat << EOF > ${METADATA_FILE} +// Generated file DO NOT EDIT +// Run download_input_files.sh +name: "android_inputs" +description: + "Assorted files downloaded from ${TZBB_PROJECT_URL}" + +third_party { + url { + type: HOMEPAGE + value: "${TZBB_PROJECT_URL}" + } +EOF + +download ${SERVER_SOURCE_BASE_URL} ${LICENSE_FILE} ${HOST_INPUTS_DIR} +download ${SERVER_RELEASE_BASE_URL}/${RELEASE_ID} ${INPUT_DATA_FILE} ${HOST_INPUTS_DIR} + +PROTO_UPGRADE_DATE=$(date +'{ year: '%Y' month: '%-m' day: '%-d' }') + +# Finish the METADATA file +cat << EOF >> ${METADATA_FILE} + version: "${RELEASE_ID}" + last_upgrade_date ${PROTO_UPGRADE_DATE} + license_type: RESTRICTED +} +EOF + +ln -sr ${HOST_INPUTS_DIR}/DATA_LICENSE ${HOST_INPUTS_DIR}/LICENSE + +# Handle the unzip step to HOST_DOWNLOADS_DIR +unzip -d ${HOST_DOWNLOADS_DIR} ${HOST_INPUTS_DIR}/${INPUT_DATA_FILE} + +echo Look in ${HOST_INPUTS_DIR} for input files.... diff --git a/android/tools/host/run_process_in_docker.sh b/android/tools/host/run_process_in_docker.sh new file mode 100755 index 0000000..bdb615a --- /dev/null +++ b/android/tools/host/run_process_in_docker.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HOST_TOOLS_DIR=$(realpath $(dirname $0)) +CONTAINER_TOOLS_DIR=$(realpath $HOST_TOOLS_DIR/../container) +TZBB_ROOT=$(realpath $HOST_TOOLS_DIR/../../..) + +DOCKER_USERNAME=$(id -un) +DOCKER_UID=$(id -u) +DOCKER_GID=$(id -g) + +echo "This may need your sudo password in order to access docker:" +set -x +sudo docker build --build-arg userid=$DOCKER_UID --build-arg groupid=$DOCKER_GID --build-arg username=$DOCKER_USERNAME --build-arg tzbbroot=$TZBB_ROOT -t android-tzbb . +sudo docker run -it --rm -v $TZBB_ROOT:/timezone-boundary-builder android-tzbb $* +set +x + diff --git a/android/tools/host/run_tzbb.sh b/android/tools/host/run_tzbb.sh new file mode 100755 index 0000000..4c30453 --- /dev/null +++ b/android/tools/host/run_tzbb.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HOST_TOOLS_DIR=$(realpath $(dirname $0)) +source ${HOST_TOOLS_DIR}/common.sh + +${HOST_TOOLS_DIR}/run_process_in_docker.sh /timezone-boundary-builder/android/tools/container/run_tzbb.sh \ + --downloads_dir ${CONTAINER_DOWNLOADS_DIR} \ + --dist_dir ${CONTAINER_DIST_DIR} \ + --skip_zip \ + --skip_shapefile \ + $* -- cgit v1.2.3